tryton-do/tryton
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add filestore module

Browse Source 
Replace custom Attachment setter/getter by a generic module that can be used by
others.

issue5686
review22141003
This commit is contained in:
Cédric Krier
2016-07-29 10:46:52 +02:00
parent 70a979449c
commit 065d1b7bb6
9 changed files with 276 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG
+1
View File
@@ -1,3 +1,4 @@
* Add filestore module
* Allow None in Greater/Less PYSON
* Add option to set admin password with trytond-admin
* Remove super password
doc/ref/filestore.rst
+42
View File
@@ -0,0 +1,42 @@
.. _ref-filestore:
.. module:: trytond.filestore
=========
FileStore
=========
.. class:: FileStore()
The class is used to store and retrieve files from the directory defined in the
configuration `path` of `database` section. It uses a two levels of directory
composed of the 2 chars of the file hash. It is an append only storage.
.. method:: get(id[, prefix])
Retrieve the content of the file referred by the id in the prefixed directory.
.. method:: getmany(ids[, prefix])
Retrieve a list of contents for the sequence of ids.
.. method:: size(id[, prefix])
Return the size of the file referred by the id in the prefixed directory.
.. method:: sizemany(ids[, prefix])
Return a list of sizes for the sequence of ids.
.. method:: set(data[, prefix])
Store the data in the prefixed directory and return the identifiers.
.. method:: setmany(data[, prefix])
Store the sequence of data and return a list of identifiers.
.. note::
The class can be overridden by setting a fully qualified name of a
alternative class defined in the configuration `class` of the `database`
section.
..
doc/ref/index.rst
+2 -1
View File
@@ -11,7 +11,8 @@ API Reference
wizard
pyson
transaction
tools/singleton
tools/index
pool
rpc
sendmail
filestore
doc/ref/tools/index.rst
+13
View File
@@ -0,0 +1,13 @@
.. _ref-tools-index:
=====
Tools
=====
Tools API reference.
.. toctree::
:maxdepth: 1
misc
singleton
doc/ref/tools/misc.rst
+10
View File
@@ -0,0 +1,10 @@
.. _ref-tools:
.. module:: trytond.tools
=============
Miscellaneous
=============
.. method:: resolve(name)
Resolve a dotted name to a global object.
trytond/filestore.py
+67
View File
@@ -0,0 +1,67 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
import os
import hashlib
from trytond.config import config
from trytond.tools import resolve
__all__ = ['filestore']
class FileStore(object):
def get(self, id, prefix=''):
filename = self._filename(id, prefix)
with open(filename, 'rb') as fp:
return fp.read()
def getmany(self, ids, prefix=''):
return [self.get(id, prefix) for id in ids]
def size(self, id, prefix=''):
filename = self._filename(id, prefix)
statinfo = os.stat(filename)
return statinfo.st_size
def sizemany(self, ids, prefix=''):
return [self.size(id, prefix) for id in ids]
def set(self, data, prefix=''):
id = self._id(data)
filename = self._filename(id, prefix)
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname, 0770)
collision = 0
while True:
basename = os.path.basename(filename)
if os.path.exists(filename):
if data != self.get(basename, prefix):
collision += 1
filename = self._filename(
'%s-%s' % (id, collision), prefix)
continue
else:
with open(filename, 'wb')as fp:
fp.write(data)
return basename
def setmany(self, data, prefix=''):
return [self.set(d, prefix) for d in data]
def _filename(self, id, prefix):
path = os.path.normpath(config.get('database', 'path'))
filename = os.path.join(path, prefix, id[0:2], id[2:4], id)
filename = os.path.normpath(filename)
if not filename.startswith(path):
raise ValueError('Bad prefix')
return filename
def _id(self, data):
return hashlib.md5(data).hexdigest()
if config.get('database', 'class'):
FileStore = resolve(config.get('database', 'class'))
filestore = FileStore()
trytond/ir/attachment.py
+26 -62
View File
@@ -1,15 +1,14 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
import os
import hashlib
from sql import Null
from sql.operators import Concat
from ..model import ModelView, ModelSQL, fields, Unique
from ..config import config
from .. import backend
from ..transaction import Transaction
from ..pyson import Eval
from .resource import ResourceMixin
from ..filestore import filestore
__all__ = [
'Attachment',
@@ -39,8 +38,7 @@ class Attachment(ResourceMixin, ModelSQL, ModelView):
link = fields.Char('Link', states={
'invisible': Eval('type') != 'link',
}, depends=['type'])
digest = fields.Char('Digest', size=32)
collision = fields.Integer('Collision')
file_id = fields.Char('File ID', readonly=True)
data_size = fields.Function(fields.Integer('Data size', states={
'invisible': Eval('type') != 'data',
}, depends=['type']), 'get_data')
@@ -77,14 +75,26 @@ class Attachment(ResourceMixin, ModelSQL, ModelView):
table.drop_column('res_model')
table.drop_column('res_id')
# Migration from 4.0: merge digest and collision into file_id
if table.column_exist('digest') and table.column_exist('collision'):
cursor.execute(*attachment.update(
[attachment.file_id],
[attachment.digest],
where=(attachment.collision == 0)
| (attachment.collision == Null)))
cursor.execute(*attachment.update(
[attachment.file_id],
[Concat(Concat(attachment.digest, '-'),
attachment.collision)],
where=(attachment.collision != 0)
& (attachment.collision != Null)))
table.drop_column('digest')
table.drop_column('collision')
@staticmethod
def default_type():
return 'data'
@staticmethod
def default_collision():
return 0
def get_data(self, name):
db_name = Transaction().database.name
format_ = Transaction().context.get(
@@ -92,22 +102,16 @@ class Attachment(ResourceMixin, ModelSQL, ModelView):
value = None
if name == 'data_size' or format_ == 'size':
value = 0
if self.digest:
filename = self.digest
if self.collision:
filename = filename + '-' + str(self.collision)
filename = os.path.join(config.get('database', 'path'), db_name,
filename[0:2], filename[2:4], filename)
if self.file_id:
if name == 'data_size' or format_ == 'size':
try:
statinfo = os.stat(filename)
value = statinfo.st_size
value = filestore.size(self.file_id, prefix=db_name)
except OSError:
pass
else:
try:
with open(filename, 'rb') as file_p:
value = fields.Binary.cast(file_p.read())
value = fields.Binary.cast(
filestore.get(self.file_id, prefix=db_name))
except IOError:
pass
return value
@@ -117,51 +121,11 @@ class Attachment(ResourceMixin, ModelSQL, ModelView):
if value is None:
return
transaction = Transaction()
cursor = transaction.connection.cursor()
table = cls.__table__()
db_name = transaction.database.name
directory = os.path.join(config.get('database', 'path'), db_name)
if not os.path.isdir(directory):
os.makedirs(directory, 0770)
digest = hashlib.md5(value).hexdigest()
directory = os.path.join(directory, digest[0:2], digest[2:4])
if not os.path.isdir(directory):
os.makedirs(directory, 0770)
filename = os.path.join(directory, digest)
collision = 0
if os.path.isfile(filename):
with open(filename, 'rb') as file_p:
data = file_p.read()
if value != data:
cursor.execute(*table.select(table.collision,
where=(table.digest == digest)
& (table.collision != 0),
group_by=table.collision,
order_by=table.collision))
collision2 = 0
for row in cursor.fetchall():
collision2 = row[0]
filename = os.path.join(directory,
digest + '-' + str(collision2))
if os.path.isfile(filename):
with open(filename, 'rb') as file_p:
data = file_p.read()
if value == data:
collision = collision2
break
if collision == 0:
collision = collision2 + 1
filename = os.path.join(directory,
digest + '-' + str(collision))
with open(filename, 'wb') as file_p:
file_p.write(value)
else:
with open(filename, 'wb') as file_p:
file_p.write(value)
file_id = filestore.set(value, prefix=db_name)
cls.write(attachments, {
'digest': digest,
'collision': collision,
})
'file_id': file_id,
})
@fields.depends('description')
def on_change_with_summary(self, name=None):
trytond/tests/test_filestore.py
+100
View File
@@ -0,0 +1,100 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
import os
import shutil
import tempfile
import unittest
from mock import patch
from trytond.config import config
from trytond.filestore import filestore
class FileStoreTestCase(unittest.TestCase):
"Test FileStore"
def setUp(self):
path = config.get('database', 'path')
dtemp = tempfile.mkdtemp()
config.set('database', 'path', dtemp)
self.addCleanup(config.set, 'database', 'path', path)
self.addCleanup(shutil.rmtree, dtemp)
def data(self):
return os.urandom(10)
def test_set(self):
"Test set"
result = filestore.set(self.data(), prefix='test')
self.assertTrue(result)
self.assertIsInstance(result, basestring)
def test_setmany(self):
"Test setmany"
result = filestore.setmany([self.data(), self.data()], prefix='test')
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
self.assertIsInstance(result[0], basestring)
def test_get(self):
"Test get"
data = self.data()
id = filestore.set(data, prefix='test')
self.assertEqual(filestore.get(id, prefix='test'), data)
def test_getmany(self):
"Test getmany"
data = [self.data(), self.data()]
ids = filestore.setmany(data, prefix='test')
self.assertListEqual(filestore.getmany(ids, prefix='test'), data)
def test_size(self):
"Test size"
data = self.data()
id = filestore.set(data, prefix='test')
self.assertEqual(filestore.size(id, prefix='test'), len(data))
def test_sizemany(self):
"Test sizemany"
data = [self.data(), self.data()]
lens = [len(d) for d in data]
ids = filestore.setmany(data, prefix='test')
self.assertListEqual(filestore.sizemany(ids, prefix='test'), lens)
def test_prefix(self):
"Test prefix"
data = self.data()
id = filestore.set(data, prefix='foo')
with self.assertRaises(IOError):
filestore.get(id, prefix='bar')
def test_bad_prefix(self):
"Test bad prefix"
with self.assertRaises(ValueError):
filestore.set(self.data(), prefix='../')
def test_duplicate(self):
"Test duplicate"
data = self.data()
id = filestore.set(data, prefix='test')
self.assertEqual(filestore.set(data, prefix='test'), id)
def test_collision(self):
"Test collision"
data1 = self.data()
data2 = self.data()
id1 = filestore.set(data1, prefix='test')
with patch.object(filestore, '_id') as _id:
_id.return_value = id1
id2 = filestore.set(data2, prefix='test')
self.assertNotEqual(id1, id2)
self.assertEqual(filestore.get(id1, prefix='test'), data1)
self.assertEqual(filestore.get(id2, prefix='test'), data2)
def suite():
return unittest.TestLoader().loadTestsFromTestCase(FileStoreTestCase)
trytond/tools/misc.py
+15
View File
@@ -12,6 +12,7 @@ from itertools import islice
import types
import io
import warnings
import importlib
from sql import Literal
from sql.operators import Or
@@ -274,3 +275,17 @@ def is_instance_method(cls, method):
type_ = klass.__dict__.get(method)
if type_ is not None:
return isinstance(type_, types.FunctionType)
def resolve(name):
"Resolve a dotted name to a global object."
name = name.split('.')
used = name.pop(0)
found = importlib.import_module(used)
for n in name:
used = used + '.' + n
try:
found = getattr(found, n)
except AttributeError:
found = importlib.import_module(used)
return found