diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG index 3a5b5e4dee..a1f7091065 100644 --- a/trytond/CHANGELOG +++ b/trytond/CHANGELOG @@ -1,3 +1,4 @@ +* Add bulk_create, bulk_delete, bulk_save and bulk_func to ModelStorage * Add editable state on field * Enforce readonly on field when checking access * Add the icon parameter to register_authentication_service diff --git a/trytond/doc/ref/models.rst b/trytond/doc/ref/models.rst index a5c4d263dd..b5cd22f904 100644 --- a/trytond/doc/ref/models.rst +++ b/trytond/doc/ref/models.rst @@ -382,6 +382,10 @@ Class methods: ``vlist`` is list of dictionaries with fields names as key and created values as value and return the list of new instances. +.. classmethod:: ModelStorage.bulk_create([batch_size[, auto]]) + + Returns a :class:`BulkBuffer` for the :meth:`~ModelStorage.create` method. + .. classmethod:: ModelStorage.read(ids, fields_names) Return a list of dictionary for the record ids. @@ -426,6 +430,10 @@ Class methods: Delete records. +.. classmethod:: ModelStorage.bulk_delete([batch_size[, auto]]) + + Returns a :class:`BulkBuffer` for the :meth:`~ModelStorage.delete` method. + .. classmethod:: ModelStorage.check_modification(mode, records[, values[, external]]) Method called after ``records`` are created with ``values`` as a list of @@ -586,6 +594,14 @@ Class methods: method must be overridden to add validation for the field names set and must raise an exception if validation fails. +.. classmethod:: ModelStorage.bulk_save([batch_size[, auto]]) + + Returns a :class:`BulkBuffer` for the :meth:`~ModelStorage.save` method. + +.. classmethod:: ModelStorage.bulk_func(name[, batch_size[, auto]]) + + Returns a :class:`BulkBuffer` for the named method. + Dual methods: .. classmethod:: ModelStorage.save(records) @@ -884,6 +900,28 @@ Additional options are available: * ``begin``: optimize for constant pattern and anchored to the beginning of the string +BulkBuffer +========== + +.. class:: BulkBuffer(function[, batch_size[, auto]]) + + A `context manager`_ that bufferizes a list of values. + When the list is greater than the ``batch_size`` and ``auto`` is ``True`` or + the context is exited, the buffer is flushed by calling the function with + the values in the list by ``batch_size``. + +.. method:: BulkBuffer.push(values) + + Push a values into the buffer. + +.. method:: BulkBuffer.extend(vlist) + + Extend the buffer by appending the values from the iterable. + +.. method:: BulkBuffer.flush() + + Call the function with the current valuess in the list and clear it. + BrowseList ========== @@ -1347,3 +1385,4 @@ Instance methods: .. _mixin: http://en.wikipedia.org/wiki/Mixin .. _JSON: http://en.wikipedia.org/wiki/Json .. _UNION: http://en.wikipedia.org/wiki/Union_(SQL)#UNION_operator +.. _`context manager`: http://docs.python.org/reference/datamodel.html#context-managers diff --git a/trytond/trytond/model/modelstorage.py b/trytond/trytond/model/modelstorage.py index de8c320041..bf2dc6c69a 100644 --- a/trytond/trytond/model/modelstorage.py +++ b/trytond/trytond/model/modelstorage.py @@ -346,6 +346,10 @@ class ModelStorage(Model): transaction.create_records[cls.__name__].extend(ids) return ids + @classmethod + def bulk_create(cls, batch_size=None, auto=True): + return BulkBuffer(cls.create, batch_size=batch_size, auto=auto) + @classmethod def _before_read(cls, ids, fields_names): pool = Pool() @@ -558,6 +562,10 @@ class ModelStorage(Model): for meth in on_delete: meth() + @classmethod + def bulk_delete(cls, batch_size=None, auto=True): + return BulkBuffer(cls.delete, batch_size=batch_size, auto=auto) + @classmethod def check_modification(cls, mode, records, values=None, external=False): assert mode in {'create', 'write', 'delete'} @@ -2322,6 +2330,59 @@ class ModelStorage(Model): record._removed = None records = latter + @classmethod + def bulk_save(cls, batch_size=None, auto=True): + return BulkBuffer(cls.save, batch_size=batch_size, auto=auto) + + @classmethod + def bulk_func(cls, name, batch_size=None, auto=True): + return BulkBuffer(getattr(cls, name), batch_size=batch_size, auto=auto) + + +class BulkBuffer: + __slots__ = ('_func', '_batch_size', '_auto', '_pending') + + def __init__(self, function, batch_size=None, auto=True): + if batch_size is None: + batch_size = record_cache_size(Transaction()) + if batch_size <= 0: + raise ValueError("batch_size must be greater than 0") + self._func = function + self._batch_size = batch_size + self._auto = auto + self._pending = [] + + def push(self, values): + self._pending.append(values) + if self._auto and len(self._pending) >= self._batch_size: + self.flush() + + def extend(self, vlist): + if self._auto: + for value in vlist: + self.push(value) + else: + self._pending.extend(vlist) + + def flush(self): + while self._pending: + batch = self._pending[:self._batch_size] + del self._pending[:self._batch_size] + self._func(batch) + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + if type is None: + self.flush() + + def __len__(self): + return len(self._pending) + + def __bool__(self): + return bool(self._pending) + class BrowseList(list): __slots__ = ( diff --git a/trytond/trytond/tests/test_modelstorage.py b/trytond/trytond/tests/test_modelstorage.py index deffd4bee1..ccad382512 100644 --- a/trytond/trytond/tests/test_modelstorage.py +++ b/trytond/trytond/tests/test_modelstorage.py @@ -107,6 +107,51 @@ class ModelStorageTestCase(DBTestCase): record = ModelStorageContext(record.id) self.assertEqual(record.context.get('foo'), 'bar') + @with_transaction() + def test_bulk_create(self): + "Test bulk create" + pool = Pool() + ModelStorage = pool.get('test.modelstorage') + + with ModelStorage.bulk_create(batch_size=2) as create: + for i in range(3): + create.push({'name': str(i)}) + + records = ModelStorage.search([]) + self.assertEqual(len(records), 3) + + @with_transaction() + def test_bulk_delete(self): + "Test bulk delete" + pool = Pool() + ModelStorage = pool.get('test.modelstorage') + + records = ModelStorage.create([{'name': str(i)} for i in range(1, 4)]) + + with ModelStorage.bulk_delete(batch_size=2) as delete: + delete.extend(records) + + records = ModelStorage.search([]) + self.assertEqual(len(records), 0) + + @with_transaction() + def test_bulk_save(self): + "Test bulk save" + pool = Pool() + ModelStorage = pool.get('test.modelstorage') + + record = ModelStorage(name="foo") + record.save() + + with ModelStorage.bulk_save(batch_size=2) as save: + record.name = "bar" + save.push(record) + for i in range(3): + save.push(ModelStorage(name=str(i))) + + records = ModelStorage.search([]) + self.assertEqual(len(records), 4) + @with_transaction() def test_save_mixed_context(self): 'Test save with mixed context '