Add BrowseList for Model instances

Closes #14808

trytond/CHANGELOG

@@ -1,3 +1,4 @@
+* Add BrowseList for Model instances
 * Remove cursor_dict
 * Add row factory to cursor
 * Add filename extension to Binary field

trytond/doc/ref/models.rst

@@ -539,7 +539,7 @@ Class methods:
 
 .. classmethod:: ModelStorage.browse(ids)
 
-   Return a list of record instance for the ``ids``.
+   Return a :class:`BrowseList` of record instance for the ``ids``.
 
 .. classmethod:: ModelStorage.export_data(records, fields_names[, header])
 
@@ -881,6 +881,23 @@ Additional options are available:
    * ``begin``: optimize for constant pattern and anchored to the beginning of
      the string
 
+BrowseList
+==========
+
+.. class:: BrowseList(Model, ids)
+
+   A class inheriting from :py:class:`list <list>` that lazily stores
+   :class:`ModelStorage` instances together.
+   The instances are created and stored when accessed.
+   The mutation methods from the list accept :class:`ModelStorage` instances or
+   :py:class:`integer <int>` and always store a new instance.
+
+Instance attributes:
+
+.. method:: BrowseList.ids
+
+   The list of ids of the instances.
+
 
 convert_from
 ------------

trytond/trytond/model/__init__.py

@@ -10,7 +10,7 @@ from .match import MatchMixin
 from .model import Model
 from .modelsingleton import ModelSingleton
 from .modelsql import Check, Exclude, Index, ModelSQL, Unique, convert_from
-from .modelstorage import EvalEnvironment, ModelStorage
+from .modelstorage import BrowseList, EvalEnvironment, ModelStorage
 from .modelview import ModelView
 from .multivalue import MultiValueMixin, ValueMixin
 from .order import sequence_ordered, sequence_reorder, sort
@@ -20,6 +20,7 @@ from .union import UnionMixin
 from .workflow import Workflow
 
 __all__ = [
+    BrowseList,
     ChatMixin,
     Check,
     DeactivableMixin,

trytond/trytond/model/fields/function.py

@@ -132,14 +132,14 @@ class Function(Field):
         multiple = self.getter_multiple(method)
 
         records = Model.browse(ids)
-        for record, value in zip(records, values):
-            assert record.id == value['id']
+        for id, value in zip(records.ids, values):
+            assert id == value['id']
             for fname, val in value.items():
                 field = Model._fields.get(fname)
                 if field and field._type not in {
                         'many2one', 'reference',
                         'one2many', 'many2many', 'one2one'}:
-                    record._local_cache[record.id][fname] = val
+                    records._local_cache[id][fname] = val
 
         def call(name):
             if not instance_method:

trytond/trytond/model/fields/many2many.py

@@ -154,7 +154,7 @@ class Many2Many(Field):
         clause += [(self.target, '!=', None)]
         if self.filter:
             clause.append((self.target, 'where', self.filter))
-        to_read = [r.id for r in Relation.search(clause, order=order)]
+        to_read = Relation.search(clause, order=order).ids
         relations = {t['id']: t
             for t in Relation.read(to_read, [self.origin, self.target])}
 

trytond/trytond/model/fields/one2many.py

@@ -162,7 +162,7 @@ class One2Many(Field):
             clause = [(self.field, 'in', ids)]
         if self.filter:
             clause.append(self.filter)
-        to_read = [r.id for r in Target.search(clause, order=order)]
+        to_read = Target.search(clause, order=order).ids
         targets = {t['id']: t
             for t in Target.read(to_read, ['id', self.field])}
 

trytond/trytond/model/modelstorage.py

@@ -8,13 +8,14 @@ import decimal
 import json
 import math
 import random
+import sys
 import time
 import warnings
 from collections import defaultdict
 from decimal import Decimal
 from functools import lru_cache
 from io import StringIO
-from itertools import chain, groupby, islice
+from itertools import chain, groupby, islice, tee
 from operator import itemgetter
 
 import trytond.config as config
@@ -36,7 +37,7 @@ from . import fields
 from .descriptors import dualmethod
 from .model import Model
 
-__all__ = ['ModelStorage', 'EvalEnvironment']
+__all__ = ['ModelStorage', 'BrowseList', 'EvalEnvironment']
 
 
 def local_cache(Model, transaction=None):
@@ -106,6 +107,13 @@ class _search_RPC(RPC):
         return args, kwargs, context, timestamp
 
 
+def _rpc_record_ids(lst):
+    if isinstance(lst, BrowseList):
+        return lst.ids
+    else:
+        return list(map(int, lst))
+
+
 class ModelStorage(Model):
     """
     Define a model with storage capability in Tryton.
@@ -146,7 +154,7 @@ class ModelStorage(Model):
                         size_limits={
                             0: request_records_limit,
                             },
-                        result=lambda r: list(map(int, r))),
+                        result=_rpc_record_ids),
                     'read': RPC(
                         timeout=request_timeout,
                         size_limits={
@@ -171,9 +179,9 @@ class ModelStorage(Model):
                         size_limits={
                             0: request_records_limit,
                             },
-                        result=lambda r: list(map(int, r))),
+                        result=_rpc_record_ids),
                     'search': _search_RPC(
-                        result=lambda r: list(map(int, r)),
+                        result=_rpc_record_ids,
                         size_limits={
                             2: request_records_limit,
                             },
@@ -895,14 +903,7 @@ class ModelStorage(Model):
         '''
         Return a list of instance for the ids
         '''
-        transaction = Transaction()
-        ids = list(map(int, ids))
-        _local_cache = local_cache(cls, transaction)
-        transaction_cache = transaction.get_cache()
-        return [cls(x, _ids=ids,
-                _local_cache=_local_cache,
-                _transaction_cache=transaction_cache,
-                _transaction=transaction) for x in ids]
+        return BrowseList(cls, map(int, ids))
 
     def __export_row(self, fields_names):
         pool = Pool()
@@ -1835,7 +1836,7 @@ class ModelStorage(Model):
             transaction = Transaction()
         self._transaction = transaction
         self._user = transaction.user
-        self._context = transaction.context
+        self._context = kwargs.pop('_context', transaction.context)
         if id is not None:
             id = int(id)
         if _ids is not None:
@@ -1991,9 +1992,13 @@ class ModelStorage(Model):
                     yield id_
         read_size = max(1, min(
                 self._cache.size_limit, self._local_cache.size_limit))
-        index = self._ids.index(self.id)
-        ids = islice(self._ids, index, index + read_size)
-        ids = unique(filter(filter_, ids))
+        try:
+            index = self._ids.index(self.id)
+        except ValueError:
+            ids = [self.id]
+        else:
+            ids = islice(self._ids, index, index + read_size)
+            ids = unique(filter(filter_, ids))
 
         kwargs_cache = {}
         pysoned_ctx = {}
@@ -2265,6 +2270,175 @@ class ModelStorage(Model):
             records = latter
 
 
+class BrowseList(list):
+    __slots__ = (
+        '_Model', '_ids',
+        '_transaction', '_context', '_local_cache', '_transaction_cache',
+        )
+
+    def __init__(self, Model, ids):
+        super().__init__()
+        assert issubclass(Model, ModelStorage)
+        self._Model = Model
+        self._ids = list(ids)
+        self._transaction = Transaction()
+        self._context = self._transaction.context
+        self._local_cache = local_cache(Model, self._transaction)
+        self._transaction_cache = self._transaction.get_cache()
+
+    @property
+    def ids(self):
+        return self._ids.copy()
+
+    def __instantiates_idx(self, start, end):
+        return (self.__instantiate_idx(i) for i in range(start, end))
+
+    def __instantiate_idx(self, index):
+        id = self._ids[index]
+        return self.__instantiate(id)
+
+    def __instantiates(self, ids):
+        return (self.__instantiate(id) for id in ids)
+
+    def __instantiate(self, id):
+        return self._Model(
+            id,
+            _ids=self._ids,
+            _local_cache=self._local_cache,
+            _transaction_cache=self._transaction_cache,
+            _transaction=self._transaction,
+            _context=self._context)
+
+    def __check_size(self, index):
+        if index >= len(self._ids):
+            raise IndexError("list index out of range")
+
+    def __fill(self, end=None):
+        if end is None:
+            end = len(self._ids) - 1
+        else:
+            end = min(end, len(self._ids) - 1)
+        length = list.__len__(self)
+        assert end < len(self._ids)
+        if end == length:
+            list.append(self, self.__instantiate_idx(end))
+        elif end > length:
+            list.extend(self, self.__instantiates_idx(length, end + 1))
+
+    def __contains__(self, item):
+        self.__fill()
+        return super().__contains__(item)
+
+    def __delitem__(self, index):
+        self.__check_size(index)
+        self.__fill(index + 1)
+        super().__delitem__(index)
+        del self._ids[index]
+
+    def __eq__(self, value):
+        return list(self) == value
+
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            size = len(self._ids)
+            end = min(index.stop or size, size) - 1
+        else:
+            end = index
+        self.__check_size(end)
+        self.__fill(end)
+        return super().__getitem__(index)
+
+    def __iter__(self):
+        for i in range(len(self._ids)):
+            if not i % 100 and i + 100 < len(self._ids):
+                self[i:i + 100]
+            yield self[i]
+
+    def __len__(self):
+        return len(self._ids)
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({self._Model!r}, {self._ids!r})'
+
+    def __setitem__(self, index, value):
+        if isinstance(index, slice):
+            size = len(self._ids)
+            end = min(index.stop or size, size) - 1
+            id = map(int, value)
+        else:
+            end = index
+            id = int(value)
+        self.__check_size(end)
+        self.__fill(end + 1)
+        self._ids[index] = id
+        if isinstance(index, slice):
+            instance = self.__instantiates(value)
+        else:
+            instance = self.__instantiate(value)
+        super().__setitem__(index, instance)
+
+    def append(self, value):
+        self.__fill()
+        self._ids.append(int(value))
+        instance = self.__instantiate(value)
+        super().append(instance)
+
+    def clear(self):
+        super().clear()
+        self._ids = []
+
+    def copy(self):
+        self.__fill()
+        return super().copy()
+
+    def count(self, value):
+        self.__fill()
+        return super().count(value)
+
+    def extend(self, iterable):
+        iterable, iterable_ids = tee(iterable)
+        self.__fill()
+        self._ids.extend(map(int, iterable_ids))
+        super().extend(self.__instantiates(iterable))
+
+    def index(self, value, start=0, stop=sys.maxsize):
+        self.__fill(min(stop, len(self._ids)) - 1)
+        return super().index(value, start, stop)
+
+    def insert(self, index, value):
+        self.__fill(min(index, len(self._ids)))
+        self._ids.insert(index, int(value))
+        instance = self.__instantiate(value)
+        super().insert(index, instance)
+
+    def pop(self, index=-1):
+        if index < 0:
+            end = len(self.ids)
+        else:
+            end = index + 1
+        self.__fill(end)
+        super().pop(index)
+        self._ids.pop(index)
+
+    def remove(self, value):
+        if not isinstance(value, self._Model):
+            raise ValueError(f"unsupported value {value!r}")
+        self.__fill(self._ids.index(value.id) + 1)
+        super().remove(value)
+        self._ids.remove(value.id)
+
+    def reverse(self):
+        self.__fill()
+        super().reverse()
+        self._ids.reverse()
+
+    def sort(self, /, *, key=None, reverse=False):
+        self.__fill()
+        super().sort(key=key, reverse=reverse)
+        self._ids.clear()
+        self._ids.extend(map(int, list.__iter__(self)))
+
+
 class EvalEnvironment(dict):
     __slots__ = ('_record', '_model')
 

trytond/trytond/tests/test_modelstorage.py

@@ -3,7 +3,7 @@
 
 import warnings
 
-from trytond.model import EvalEnvironment
+from trytond.model import BrowseList, EvalEnvironment
 from trytond.model.exceptions import (
     AccessError, DomainValidationError, RequiredValidationError)
 from trytond.model.modelstorage import _UnsavedRecordError
@@ -768,6 +768,301 @@ class ModelStorageTestCase(TestCase):
         self.assertEqual(notification.description, "Long description")
 
 
+class BrowseListTestCase(TestCase):
+    "Test BrowseList"
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        activate_module('tests')
+
+    @with_transaction()
+    def test_contains(self):
+        "Test __contains__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertIn(Model(2), blist)
+
+    @with_transaction()
+    def test_delitem(self):
+        "Test __delitem__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        del blist[1]
+
+        self.assertEqual(len(blist), 2)
+        self.assertEqual(blist.ids, [1, 3])
+        self.assertEqual(blist[0]._ids, blist.ids)
+
+    @with_transaction()
+    def test_eq(self):
+        "Test __eq__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertEqual(blist, [Model(1), Model(2), Model(3)])
+
+    @with_transaction()
+    def test_getitem(self):
+        "Test __getitem__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertEqual(blist[0], Model(1))
+        self.assertEqual(blist[1], Model(2))
+        self.assertEqual(blist[2], Model(3))
+        with self.assertRaises(IndexError):
+            blist[42]
+
+    @with_transaction()
+    def test_getitem_slice(self):
+        "Test __getitem__ with slice"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertEqual(blist[:0], [])
+        self.assertEqual(blist[:10], [Model(1), Model(2), Model(3)])
+        self.assertEqual(blist[1:3], [Model(2), Model(3)])
+        self.assertEqual(blist[10:20], [])
+
+    @with_transaction()
+    def test_iter(self):
+        "Test __iter__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertEqual(blist, [Model(i) for i in range(1, 4)])
+
+    @with_transaction()
+    def test_len(self):
+        "Test __len__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertEqual(list.__len__(blist), 0)
+        self.assertEqual(len(blist), 3)
+
+        list(blist)
+        self.assertEqual(list.__len__(blist), 3)
+        self.assertEqual(len(blist), 3)
+
+    @with_transaction()
+    def test_repr(self):
+        "Test __repr__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2])
+
+        self.assertEqual(
+            repr(blist),
+            "BrowseList(<class 'trytond.pool.test.modelstorage'>, [1, 2])")
+
+    @with_transaction()
+    def test_setitem(self):
+        "Test __setitem__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        blist[1] = Model(42)
+
+        self.assertEqual(blist[1], Model(42))
+        self.assertEqual(blist.ids, [1, 42, 3])
+        self.assertEqual(blist[0]._ids, blist.ids)
+
+    @with_transaction()
+    def test_setitem_slice(self):
+        "Test __setitem__ slice"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3, 4, 5])
+
+        blist[1:4] = [Model(42)]
+
+        self.assertEqual(blist, [Model(1), Model(42), Model(5)])
+        self.assertEqual(blist.ids, [1, 42, 5])
+
+    @with_transaction()
+    def test_not_contains(self):
+        "Test not __contains__"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertNotIn(Model(42), blist)
+
+    @with_transaction()
+    def test_append(self):
+        "Test append"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1])
+        blist.append(Model(2))
+
+        self.assertEqual(blist, [Model(1), Model(2)])
+
+    @with_transaction()
+    def test_clear(self):
+        "Test clear"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        blist.clear()
+        self.assertEqual(blist, [])
+        self.assertEqual(blist.ids, [])
+
+    @with_transaction()
+    def test_copy(self):
+        "Test copy"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        blist_copy = blist.copy()
+        self.assertEqual(blist, blist_copy)
+        self.assertIsInstance(blist_copy, list)
+
+    @with_transaction()
+    def test_count(self):
+        "Test count"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3, 1])
+
+        self.assertEqual(blist.count(Model(1)), 2)
+
+    @with_transaction()
+    def test_extend(self):
+        "Test extend"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1])
+        blist.extend([Model(2), Model(3)])
+
+        self.assertEqual(blist, [Model(1), Model(2), Model(3)])
+        self.assertEqual(blist.ids, [1, 2, 3])
+
+    @with_transaction()
+    def test_index(self):
+        "Test index"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+
+        self.assertEqual(blist.index(Model(2)), 1)
+
+    @with_transaction()
+    def test_index_stop(self):
+        "Test index with stop"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3, 4, 5])
+
+        with self.assertRaises(ValueError):
+            blist.index(Model(4), 0, 2)
+        self.assertEqual(list.__len__(blist), 2)
+
+    @with_transaction()
+    def test_insert(self):
+        "Test insert"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+        blist.insert(2, Model(42))
+
+        self.assertEqual(blist, [Model(1), Model(2), Model(42), Model(3)])
+        self.assertEqual(blist.ids, [1, 2, 42, 3])
+
+    @with_transaction()
+    def test_pop(self):
+        "Test pop"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+        blist.pop(1)
+
+        self.assertEqual(blist, [Model(1), Model(3)])
+        self.assertEqual(blist.ids, [1, 3])
+
+    @with_transaction()
+    def test_pop_negative_index(self):
+        "Test pop negative index"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+        blist.pop(-2)
+
+        self.assertEqual(blist, [Model(1), Model(3)])
+        self.assertEqual(blist.ids, [1, 3])
+
+    @with_transaction()
+    def test_remove(self):
+        "Test remove"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+        blist.remove(Model(2))
+
+        self.assertEqual(blist, [Model(1), Model(3)])
+        self.assertEqual(blist.ids, [1, 3])
+
+    @with_transaction()
+    def test_reverse(self):
+        "Test reverse"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [1, 2, 3])
+        blist.reverse()
+
+        self.assertEqual(blist, [Model(3), Model(2), Model(1)])
+        self.assertEqual(blist.ids, [3, 2, 1])
+
+    @with_transaction()
+    def test_sort(self):
+        "Test sort"
+        pool = Pool()
+        Model = pool.get('test.modelstorage')
+
+        blist = BrowseList(Model, [2, 3, 1])
+        blist.sort()
+
+        self.assertEqual(blist, [Model(1), Model(2), Model(3)])
+        self.assertEqual(blist.ids, [1, 2, 3])
+
+
 class EvalEnvironmentTestCase(TestCase):
     "Test EvalEnvironment"