Add bulk_create, bulk_delete, bulk_save and bulk_func to ModelStorage

trytond/CHANGELOG

@@ -1,3 +1,4 @@
+* Add bulk_create, bulk_delete, bulk_save and bulk_func to ModelStorage
 * Add editable state on field
 * Enforce readonly on field when checking access
 * Add the icon parameter to register_authentication_service

trytond/doc/ref/models.rst

@@ -382,6 +382,10 @@ Class methods:
    ``vlist`` is list of dictionaries with fields names as key and created
    values as value and return the list of new instances.
 
+.. classmethod:: ModelStorage.bulk_create([batch_size[, auto]])
+
+   Returns a :class:`BulkBuffer` for the :meth:`~ModelStorage.create` method.
+
 .. classmethod:: ModelStorage.read(ids, fields_names)
 
    Return a list of dictionary for the record ids.
@@ -426,6 +430,10 @@ Class methods:
 
    Delete records.
 
+.. classmethod:: ModelStorage.bulk_delete([batch_size[, auto]])
+
+   Returns a :class:`BulkBuffer` for the :meth:`~ModelStorage.delete` method.
+
 .. classmethod:: ModelStorage.check_modification(mode, records[, values[, external]])
 
    Method called after ``records`` are created with ``values`` as a list of
@@ -586,6 +594,14 @@ Class methods:
    method must be overridden to add validation for the field names set and must
    raise an exception if validation fails.
 
+.. classmethod:: ModelStorage.bulk_save([batch_size[, auto]])
+
+   Returns a :class:`BulkBuffer` for the :meth:`~ModelStorage.save` method.
+
+.. classmethod:: ModelStorage.bulk_func(name[, batch_size[, auto]])
+
+   Returns a :class:`BulkBuffer` for the named method.
+
 Dual methods:
 
 .. classmethod:: ModelStorage.save(records)
@@ -884,6 +900,28 @@ Additional options are available:
    * ``begin``: optimize for constant pattern and anchored to the beginning of
      the string
 
+BulkBuffer
+==========
+
+.. class:: BulkBuffer(function[, batch_size[, auto]])
+
+   A `context manager`_ that bufferizes a list of values.
+   When the list is greater than the ``batch_size`` and ``auto`` is ``True`` or
+   the context is exited, the buffer is flushed by calling the function with
+   the values in the list by ``batch_size``.
+
+.. method:: BulkBuffer.push(values)
+
+   Push a values into the buffer.
+
+.. method:: BulkBuffer.extend(vlist)
+
+   Extend the buffer by appending the values from the iterable.
+
+.. method:: BulkBuffer.flush()
+
+   Call the function with the current valuess in the list and clear it.
+
 BrowseList
 ==========
 
@@ -1347,3 +1385,4 @@ Instance methods:
 .. _mixin: http://en.wikipedia.org/wiki/Mixin
 .. _JSON: http://en.wikipedia.org/wiki/Json
 .. _UNION: http://en.wikipedia.org/wiki/Union_(SQL)#UNION_operator
+.. _`context manager`: http://docs.python.org/reference/datamodel.html#context-managers

trytond/trytond/model/modelstorage.py

@@ -346,6 +346,10 @@ class ModelStorage(Model):
             transaction.create_records[cls.__name__].extend(ids)
             return ids
 
+    @classmethod
+    def bulk_create(cls, batch_size=None, auto=True):
+        return BulkBuffer(cls.create, batch_size=batch_size, auto=auto)
+
     @classmethod
     def _before_read(cls, ids, fields_names):
         pool = Pool()
@@ -558,6 +562,10 @@ class ModelStorage(Model):
         for meth in on_delete:
             meth()
 
+    @classmethod
+    def bulk_delete(cls, batch_size=None, auto=True):
+        return BulkBuffer(cls.delete, batch_size=batch_size, auto=auto)
+
     @classmethod
     def check_modification(cls, mode, records, values=None, external=False):
         assert mode in {'create', 'write', 'delete'}
@@ -2322,6 +2330,59 @@ class ModelStorage(Model):
                 record._removed = None
             records = latter
 
+    @classmethod
+    def bulk_save(cls, batch_size=None, auto=True):
+        return BulkBuffer(cls.save, batch_size=batch_size, auto=auto)
+
+    @classmethod
+    def bulk_func(cls, name, batch_size=None, auto=True):
+        return BulkBuffer(getattr(cls, name), batch_size=batch_size, auto=auto)
+
+
+class BulkBuffer:
+    __slots__ = ('_func', '_batch_size', '_auto', '_pending')
+
+    def __init__(self, function, batch_size=None, auto=True):
+        if batch_size is None:
+            batch_size = record_cache_size(Transaction())
+        if batch_size <= 0:
+            raise ValueError("batch_size must be greater than 0")
+        self._func = function
+        self._batch_size = batch_size
+        self._auto = auto
+        self._pending = []
+
+    def push(self, values):
+        self._pending.append(values)
+        if self._auto and len(self._pending) >= self._batch_size:
+            self.flush()
+
+    def extend(self, vlist):
+        if self._auto:
+            for value in vlist:
+                self.push(value)
+        else:
+            self._pending.extend(vlist)
+
+    def flush(self):
+        while self._pending:
+            batch = self._pending[:self._batch_size]
+            del self._pending[:self._batch_size]
+            self._func(batch)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        if type is None:
+            self.flush()
+
+    def __len__(self):
+        return len(self._pending)
+
+    def __bool__(self):
+        return bool(self._pending)
+
 
 class BrowseList(list):
     __slots__ = (

trytond/trytond/tests/test_modelstorage.py

@@ -107,6 +107,51 @@ class ModelStorageTestCase(DBTestCase):
             record = ModelStorageContext(record.id)
             self.assertEqual(record.context.get('foo'), 'bar')
 
+    @with_transaction()
+    def test_bulk_create(self):
+        "Test bulk create"
+        pool = Pool()
+        ModelStorage = pool.get('test.modelstorage')
+
+        with ModelStorage.bulk_create(batch_size=2) as create:
+            for i in range(3):
+                create.push({'name': str(i)})
+
+        records = ModelStorage.search([])
+        self.assertEqual(len(records), 3)
+
+    @with_transaction()
+    def test_bulk_delete(self):
+        "Test bulk delete"
+        pool = Pool()
+        ModelStorage = pool.get('test.modelstorage')
+
+        records = ModelStorage.create([{'name': str(i)} for i in range(1, 4)])
+
+        with ModelStorage.bulk_delete(batch_size=2) as delete:
+            delete.extend(records)
+
+        records = ModelStorage.search([])
+        self.assertEqual(len(records), 0)
+
+    @with_transaction()
+    def test_bulk_save(self):
+        "Test bulk save"
+        pool = Pool()
+        ModelStorage = pool.get('test.modelstorage')
+
+        record = ModelStorage(name="foo")
+        record.save()
+
+        with ModelStorage.bulk_save(batch_size=2) as save:
+            record.name = "bar"
+            save.push(record)
+            for i in range(3):
+                save.push(ModelStorage(name=str(i)))
+
+        records = ModelStorage.search([])
+        self.assertEqual(len(records), 4)
+
     @with_transaction()
     def test_save_mixed_context(self):
         'Test save with mixed context '