From 065d1b7bb6ab3ceac0333931d3528f6d5714db46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Fri, 29 Jul 2016 10:46:52 +0200 Subject: [PATCH] Add filestore module Replace custom Attachment setter/getter by a generic module that can be used by others. issue5686 review22141003 --- CHANGELOG | 1 + doc/ref/filestore.rst | 42 ++++++++++++++ doc/ref/index.rst | 3 +- doc/ref/tools/index.rst | 13 +++++ doc/ref/tools/misc.rst | 10 ++++ trytond/filestore.py | 67 +++++++++++++++++++++ trytond/ir/attachment.py | 88 +++++++++------------------- trytond/tests/test_filestore.py | 100 ++++++++++++++++++++++++++++++++ trytond/tools/misc.py | 15 +++++ 9 files changed, 276 insertions(+), 63 deletions(-) create mode 100644 doc/ref/filestore.rst create mode 100644 doc/ref/tools/index.rst create mode 100644 doc/ref/tools/misc.rst create mode 100644 trytond/filestore.py create mode 100644 trytond/tests/test_filestore.py diff --git a/CHANGELOG b/CHANGELOG index 0b870306dc..18563250e6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,4 @@ +* Add filestore module * Allow None in Greater/Less PYSON * Add option to set admin password with trytond-admin * Remove super password diff --git a/doc/ref/filestore.rst b/doc/ref/filestore.rst new file mode 100644 index 0000000000..c20bd02224 --- /dev/null +++ b/doc/ref/filestore.rst @@ -0,0 +1,42 @@ +.. _ref-filestore: +.. module:: trytond.filestore + +========= +FileStore +========= + +.. class:: FileStore() + +The class is used to store and retrieve files from the directory defined in the +configuration `path` of `database` section. It uses a two levels of directory +composed of the 2 chars of the file hash. It is an append only storage. + +.. method:: get(id[, prefix]) + +Retrieve the content of the file referred by the id in the prefixed directory. + +.. method:: getmany(ids[, prefix]) + +Retrieve a list of contents for the sequence of ids. + +.. method:: size(id[, prefix]) + +Return the size of the file referred by the id in the prefixed directory. + +.. method:: sizemany(ids[, prefix]) + +Return a list of sizes for the sequence of ids. + +.. method:: set(data[, prefix]) + +Store the data in the prefixed directory and return the identifiers. + +.. method:: setmany(data[, prefix]) + +Store the sequence of data and return a list of identifiers. + +.. note:: + The class can be overridden by setting a fully qualified name of a + alternative class defined in the configuration `class` of the `database` + section. +.. diff --git a/doc/ref/index.rst b/doc/ref/index.rst index 4203034818..1374bfb34f 100644 --- a/doc/ref/index.rst +++ b/doc/ref/index.rst @@ -11,7 +11,8 @@ API Reference wizard pyson transaction - tools/singleton + tools/index pool rpc sendmail + filestore diff --git a/doc/ref/tools/index.rst b/doc/ref/tools/index.rst new file mode 100644 index 0000000000..cbf985e9a0 --- /dev/null +++ b/doc/ref/tools/index.rst @@ -0,0 +1,13 @@ +.. _ref-tools-index: + +===== +Tools +===== + +Tools API reference. + +.. toctree:: + :maxdepth: 1 + + misc + singleton diff --git a/doc/ref/tools/misc.rst b/doc/ref/tools/misc.rst new file mode 100644 index 0000000000..ac195d0057 --- /dev/null +++ b/doc/ref/tools/misc.rst @@ -0,0 +1,10 @@ +.. _ref-tools: +.. module:: trytond.tools + +============= +Miscellaneous +============= + +.. method:: resolve(name) + +Resolve a dotted name to a global object. diff --git a/trytond/filestore.py b/trytond/filestore.py new file mode 100644 index 0000000000..59fa8db442 --- /dev/null +++ b/trytond/filestore.py @@ -0,0 +1,67 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. +import os +import hashlib + +from trytond.config import config +from trytond.tools import resolve + +__all__ = ['filestore'] + + +class FileStore(object): + + def get(self, id, prefix=''): + filename = self._filename(id, prefix) + with open(filename, 'rb') as fp: + return fp.read() + + def getmany(self, ids, prefix=''): + return [self.get(id, prefix) for id in ids] + + def size(self, id, prefix=''): + filename = self._filename(id, prefix) + statinfo = os.stat(filename) + return statinfo.st_size + + def sizemany(self, ids, prefix=''): + return [self.size(id, prefix) for id in ids] + + def set(self, data, prefix=''): + id = self._id(data) + filename = self._filename(id, prefix) + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname, 0770) + + collision = 0 + while True: + basename = os.path.basename(filename) + if os.path.exists(filename): + if data != self.get(basename, prefix): + collision += 1 + filename = self._filename( + '%s-%s' % (id, collision), prefix) + continue + else: + with open(filename, 'wb')as fp: + fp.write(data) + return basename + + def setmany(self, data, prefix=''): + return [self.set(d, prefix) for d in data] + + def _filename(self, id, prefix): + path = os.path.normpath(config.get('database', 'path')) + filename = os.path.join(path, prefix, id[0:2], id[2:4], id) + filename = os.path.normpath(filename) + if not filename.startswith(path): + raise ValueError('Bad prefix') + return filename + + def _id(self, data): + return hashlib.md5(data).hexdigest() + +if config.get('database', 'class'): + FileStore = resolve(config.get('database', 'class')) +filestore = FileStore() diff --git a/trytond/ir/attachment.py b/trytond/ir/attachment.py index dcf4f59938..1811e06789 100644 --- a/trytond/ir/attachment.py +++ b/trytond/ir/attachment.py @@ -1,15 +1,14 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -import os -import hashlib +from sql import Null from sql.operators import Concat from ..model import ModelView, ModelSQL, fields, Unique -from ..config import config from .. import backend from ..transaction import Transaction from ..pyson import Eval from .resource import ResourceMixin +from ..filestore import filestore __all__ = [ 'Attachment', @@ -39,8 +38,7 @@ class Attachment(ResourceMixin, ModelSQL, ModelView): link = fields.Char('Link', states={ 'invisible': Eval('type') != 'link', }, depends=['type']) - digest = fields.Char('Digest', size=32) - collision = fields.Integer('Collision') + file_id = fields.Char('File ID', readonly=True) data_size = fields.Function(fields.Integer('Data size', states={ 'invisible': Eval('type') != 'data', }, depends=['type']), 'get_data') @@ -77,14 +75,26 @@ class Attachment(ResourceMixin, ModelSQL, ModelView): table.drop_column('res_model') table.drop_column('res_id') + # Migration from 4.0: merge digest and collision into file_id + if table.column_exist('digest') and table.column_exist('collision'): + cursor.execute(*attachment.update( + [attachment.file_id], + [attachment.digest], + where=(attachment.collision == 0) + | (attachment.collision == Null))) + cursor.execute(*attachment.update( + [attachment.file_id], + [Concat(Concat(attachment.digest, '-'), + attachment.collision)], + where=(attachment.collision != 0) + & (attachment.collision != Null))) + table.drop_column('digest') + table.drop_column('collision') + @staticmethod def default_type(): return 'data' - @staticmethod - def default_collision(): - return 0 - def get_data(self, name): db_name = Transaction().database.name format_ = Transaction().context.get( @@ -92,22 +102,16 @@ class Attachment(ResourceMixin, ModelSQL, ModelView): value = None if name == 'data_size' or format_ == 'size': value = 0 - if self.digest: - filename = self.digest - if self.collision: - filename = filename + '-' + str(self.collision) - filename = os.path.join(config.get('database', 'path'), db_name, - filename[0:2], filename[2:4], filename) + if self.file_id: if name == 'data_size' or format_ == 'size': try: - statinfo = os.stat(filename) - value = statinfo.st_size + value = filestore.size(self.file_id, prefix=db_name) except OSError: pass else: try: - with open(filename, 'rb') as file_p: - value = fields.Binary.cast(file_p.read()) + value = fields.Binary.cast( + filestore.get(self.file_id, prefix=db_name)) except IOError: pass return value @@ -117,51 +121,11 @@ class Attachment(ResourceMixin, ModelSQL, ModelView): if value is None: return transaction = Transaction() - cursor = transaction.connection.cursor() - table = cls.__table__() db_name = transaction.database.name - directory = os.path.join(config.get('database', 'path'), db_name) - if not os.path.isdir(directory): - os.makedirs(directory, 0770) - digest = hashlib.md5(value).hexdigest() - directory = os.path.join(directory, digest[0:2], digest[2:4]) - if not os.path.isdir(directory): - os.makedirs(directory, 0770) - filename = os.path.join(directory, digest) - collision = 0 - if os.path.isfile(filename): - with open(filename, 'rb') as file_p: - data = file_p.read() - if value != data: - cursor.execute(*table.select(table.collision, - where=(table.digest == digest) - & (table.collision != 0), - group_by=table.collision, - order_by=table.collision)) - collision2 = 0 - for row in cursor.fetchall(): - collision2 = row[0] - filename = os.path.join(directory, - digest + '-' + str(collision2)) - if os.path.isfile(filename): - with open(filename, 'rb') as file_p: - data = file_p.read() - if value == data: - collision = collision2 - break - if collision == 0: - collision = collision2 + 1 - filename = os.path.join(directory, - digest + '-' + str(collision)) - with open(filename, 'wb') as file_p: - file_p.write(value) - else: - with open(filename, 'wb') as file_p: - file_p.write(value) + file_id = filestore.set(value, prefix=db_name) cls.write(attachments, { - 'digest': digest, - 'collision': collision, - }) + 'file_id': file_id, + }) @fields.depends('description') def on_change_with_summary(self, name=None): diff --git a/trytond/tests/test_filestore.py b/trytond/tests/test_filestore.py new file mode 100644 index 0000000000..46c82a16da --- /dev/null +++ b/trytond/tests/test_filestore.py @@ -0,0 +1,100 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. +import os +import shutil +import tempfile +import unittest +from mock import patch + +from trytond.config import config +from trytond.filestore import filestore + + +class FileStoreTestCase(unittest.TestCase): + "Test FileStore" + + def setUp(self): + path = config.get('database', 'path') + dtemp = tempfile.mkdtemp() + config.set('database', 'path', dtemp) + self.addCleanup(config.set, 'database', 'path', path) + self.addCleanup(shutil.rmtree, dtemp) + + def data(self): + return os.urandom(10) + + def test_set(self): + "Test set" + result = filestore.set(self.data(), prefix='test') + self.assertTrue(result) + self.assertIsInstance(result, basestring) + + def test_setmany(self): + "Test setmany" + result = filestore.setmany([self.data(), self.data()], prefix='test') + self.assertIsInstance(result, list) + self.assertEqual(len(result), 2) + self.assertIsInstance(result[0], basestring) + + def test_get(self): + "Test get" + data = self.data() + id = filestore.set(data, prefix='test') + self.assertEqual(filestore.get(id, prefix='test'), data) + + def test_getmany(self): + "Test getmany" + data = [self.data(), self.data()] + ids = filestore.setmany(data, prefix='test') + self.assertListEqual(filestore.getmany(ids, prefix='test'), data) + + def test_size(self): + "Test size" + data = self.data() + id = filestore.set(data, prefix='test') + self.assertEqual(filestore.size(id, prefix='test'), len(data)) + + def test_sizemany(self): + "Test sizemany" + data = [self.data(), self.data()] + lens = [len(d) for d in data] + ids = filestore.setmany(data, prefix='test') + self.assertListEqual(filestore.sizemany(ids, prefix='test'), lens) + + def test_prefix(self): + "Test prefix" + data = self.data() + id = filestore.set(data, prefix='foo') + with self.assertRaises(IOError): + filestore.get(id, prefix='bar') + + def test_bad_prefix(self): + "Test bad prefix" + with self.assertRaises(ValueError): + filestore.set(self.data(), prefix='../') + + def test_duplicate(self): + "Test duplicate" + data = self.data() + id = filestore.set(data, prefix='test') + self.assertEqual(filestore.set(data, prefix='test'), id) + + def test_collision(self): + "Test collision" + data1 = self.data() + data2 = self.data() + + id1 = filestore.set(data1, prefix='test') + + with patch.object(filestore, '_id') as _id: + _id.return_value = id1 + + id2 = filestore.set(data2, prefix='test') + + self.assertNotEqual(id1, id2) + self.assertEqual(filestore.get(id1, prefix='test'), data1) + self.assertEqual(filestore.get(id2, prefix='test'), data2) + + +def suite(): + return unittest.TestLoader().loadTestsFromTestCase(FileStoreTestCase) diff --git a/trytond/tools/misc.py b/trytond/tools/misc.py index b1ada39f30..4d8bc0610e 100644 --- a/trytond/tools/misc.py +++ b/trytond/tools/misc.py @@ -12,6 +12,7 @@ from itertools import islice import types import io import warnings +import importlib from sql import Literal from sql.operators import Or @@ -274,3 +275,17 @@ def is_instance_method(cls, method): type_ = klass.__dict__.get(method) if type_ is not None: return isinstance(type_, types.FunctionType) + + +def resolve(name): + "Resolve a dotted name to a global object." + name = name.split('.') + used = name.pop(0) + found = importlib.import_module(used) + for n in name: + used = used + '.' + n + try: + found = getattr(found, n) + except AttributeError: + found = importlib.import_module(used) + return found