diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG index 10eeea4ea1..fd28d83ec9 100644 --- a/trytond/CHANGELOG +++ b/trytond/CHANGELOG @@ -1,3 +1,4 @@ +* Add BrowseList for Model instances * Remove cursor_dict * Add row factory to cursor * Add filename extension to Binary field diff --git a/trytond/doc/ref/models.rst b/trytond/doc/ref/models.rst index 9c61ae8fd0..1de7f08746 100644 --- a/trytond/doc/ref/models.rst +++ b/trytond/doc/ref/models.rst @@ -539,7 +539,7 @@ Class methods: .. classmethod:: ModelStorage.browse(ids) - Return a list of record instance for the ``ids``. + Return a :class:`BrowseList` of record instance for the ``ids``. .. classmethod:: ModelStorage.export_data(records, fields_names[, header]) @@ -881,6 +881,23 @@ Additional options are available: * ``begin``: optimize for constant pattern and anchored to the beginning of the string +BrowseList +========== + +.. class:: BrowseList(Model, ids) + + A class inheriting from :py:class:`list ` that lazily stores + :class:`ModelStorage` instances together. + The instances are created and stored when accessed. + The mutation methods from the list accept :class:`ModelStorage` instances or + :py:class:`integer ` and always store a new instance. + +Instance attributes: + +.. method:: BrowseList.ids + + The list of ids of the instances. + convert_from ------------ diff --git a/trytond/trytond/model/__init__.py b/trytond/trytond/model/__init__.py index 737aba9d33..bffe60de86 100644 --- a/trytond/trytond/model/__init__.py +++ b/trytond/trytond/model/__init__.py @@ -10,7 +10,7 @@ from .match import MatchMixin from .model import Model from .modelsingleton import ModelSingleton from .modelsql import Check, Exclude, Index, ModelSQL, Unique, convert_from -from .modelstorage import EvalEnvironment, ModelStorage +from .modelstorage import BrowseList, EvalEnvironment, ModelStorage from .modelview import ModelView from .multivalue import MultiValueMixin, ValueMixin from .order import sequence_ordered, sequence_reorder, sort @@ -20,6 +20,7 @@ from .union import UnionMixin from .workflow import Workflow __all__ = [ + BrowseList, ChatMixin, Check, DeactivableMixin, diff --git a/trytond/trytond/model/fields/function.py b/trytond/trytond/model/fields/function.py index 6b2c0eb04d..8b601d9717 100644 --- a/trytond/trytond/model/fields/function.py +++ b/trytond/trytond/model/fields/function.py @@ -132,14 +132,14 @@ class Function(Field): multiple = self.getter_multiple(method) records = Model.browse(ids) - for record, value in zip(records, values): - assert record.id == value['id'] + for id, value in zip(records.ids, values): + assert id == value['id'] for fname, val in value.items(): field = Model._fields.get(fname) if field and field._type not in { 'many2one', 'reference', 'one2many', 'many2many', 'one2one'}: - record._local_cache[record.id][fname] = val + records._local_cache[id][fname] = val def call(name): if not instance_method: diff --git a/trytond/trytond/model/fields/many2many.py b/trytond/trytond/model/fields/many2many.py index 1d41d2ec37..412acd6c2b 100644 --- a/trytond/trytond/model/fields/many2many.py +++ b/trytond/trytond/model/fields/many2many.py @@ -154,7 +154,7 @@ class Many2Many(Field): clause += [(self.target, '!=', None)] if self.filter: clause.append((self.target, 'where', self.filter)) - to_read = [r.id for r in Relation.search(clause, order=order)] + to_read = Relation.search(clause, order=order).ids relations = {t['id']: t for t in Relation.read(to_read, [self.origin, self.target])} diff --git a/trytond/trytond/model/fields/one2many.py b/trytond/trytond/model/fields/one2many.py index 239b113414..9a78abf207 100644 --- a/trytond/trytond/model/fields/one2many.py +++ b/trytond/trytond/model/fields/one2many.py @@ -162,7 +162,7 @@ class One2Many(Field): clause = [(self.field, 'in', ids)] if self.filter: clause.append(self.filter) - to_read = [r.id for r in Target.search(clause, order=order)] + to_read = Target.search(clause, order=order).ids targets = {t['id']: t for t in Target.read(to_read, ['id', self.field])} diff --git a/trytond/trytond/model/modelstorage.py b/trytond/trytond/model/modelstorage.py index 661e515db5..cbf12b15b4 100644 --- a/trytond/trytond/model/modelstorage.py +++ b/trytond/trytond/model/modelstorage.py @@ -8,13 +8,14 @@ import decimal import json import math import random +import sys import time import warnings from collections import defaultdict from decimal import Decimal from functools import lru_cache from io import StringIO -from itertools import chain, groupby, islice +from itertools import chain, groupby, islice, tee from operator import itemgetter import trytond.config as config @@ -36,7 +37,7 @@ from . import fields from .descriptors import dualmethod from .model import Model -__all__ = ['ModelStorage', 'EvalEnvironment'] +__all__ = ['ModelStorage', 'BrowseList', 'EvalEnvironment'] def local_cache(Model, transaction=None): @@ -106,6 +107,13 @@ class _search_RPC(RPC): return args, kwargs, context, timestamp +def _rpc_record_ids(lst): + if isinstance(lst, BrowseList): + return lst.ids + else: + return list(map(int, lst)) + + class ModelStorage(Model): """ Define a model with storage capability in Tryton. @@ -146,7 +154,7 @@ class ModelStorage(Model): size_limits={ 0: request_records_limit, }, - result=lambda r: list(map(int, r))), + result=_rpc_record_ids), 'read': RPC( timeout=request_timeout, size_limits={ @@ -171,9 +179,9 @@ class ModelStorage(Model): size_limits={ 0: request_records_limit, }, - result=lambda r: list(map(int, r))), + result=_rpc_record_ids), 'search': _search_RPC( - result=lambda r: list(map(int, r)), + result=_rpc_record_ids, size_limits={ 2: request_records_limit, }, @@ -895,14 +903,7 @@ class ModelStorage(Model): ''' Return a list of instance for the ids ''' - transaction = Transaction() - ids = list(map(int, ids)) - _local_cache = local_cache(cls, transaction) - transaction_cache = transaction.get_cache() - return [cls(x, _ids=ids, - _local_cache=_local_cache, - _transaction_cache=transaction_cache, - _transaction=transaction) for x in ids] + return BrowseList(cls, map(int, ids)) def __export_row(self, fields_names): pool = Pool() @@ -1835,7 +1836,7 @@ class ModelStorage(Model): transaction = Transaction() self._transaction = transaction self._user = transaction.user - self._context = transaction.context + self._context = kwargs.pop('_context', transaction.context) if id is not None: id = int(id) if _ids is not None: @@ -1991,9 +1992,13 @@ class ModelStorage(Model): yield id_ read_size = max(1, min( self._cache.size_limit, self._local_cache.size_limit)) - index = self._ids.index(self.id) - ids = islice(self._ids, index, index + read_size) - ids = unique(filter(filter_, ids)) + try: + index = self._ids.index(self.id) + except ValueError: + ids = [self.id] + else: + ids = islice(self._ids, index, index + read_size) + ids = unique(filter(filter_, ids)) kwargs_cache = {} pysoned_ctx = {} @@ -2265,6 +2270,175 @@ class ModelStorage(Model): records = latter +class BrowseList(list): + __slots__ = ( + '_Model', '_ids', + '_transaction', '_context', '_local_cache', '_transaction_cache', + ) + + def __init__(self, Model, ids): + super().__init__() + assert issubclass(Model, ModelStorage) + self._Model = Model + self._ids = list(ids) + self._transaction = Transaction() + self._context = self._transaction.context + self._local_cache = local_cache(Model, self._transaction) + self._transaction_cache = self._transaction.get_cache() + + @property + def ids(self): + return self._ids.copy() + + def __instantiates_idx(self, start, end): + return (self.__instantiate_idx(i) for i in range(start, end)) + + def __instantiate_idx(self, index): + id = self._ids[index] + return self.__instantiate(id) + + def __instantiates(self, ids): + return (self.__instantiate(id) for id in ids) + + def __instantiate(self, id): + return self._Model( + id, + _ids=self._ids, + _local_cache=self._local_cache, + _transaction_cache=self._transaction_cache, + _transaction=self._transaction, + _context=self._context) + + def __check_size(self, index): + if index >= len(self._ids): + raise IndexError("list index out of range") + + def __fill(self, end=None): + if end is None: + end = len(self._ids) - 1 + else: + end = min(end, len(self._ids) - 1) + length = list.__len__(self) + assert end < len(self._ids) + if end == length: + list.append(self, self.__instantiate_idx(end)) + elif end > length: + list.extend(self, self.__instantiates_idx(length, end + 1)) + + def __contains__(self, item): + self.__fill() + return super().__contains__(item) + + def __delitem__(self, index): + self.__check_size(index) + self.__fill(index + 1) + super().__delitem__(index) + del self._ids[index] + + def __eq__(self, value): + return list(self) == value + + def __getitem__(self, index): + if isinstance(index, slice): + size = len(self._ids) + end = min(index.stop or size, size) - 1 + else: + end = index + self.__check_size(end) + self.__fill(end) + return super().__getitem__(index) + + def __iter__(self): + for i in range(len(self._ids)): + if not i % 100 and i + 100 < len(self._ids): + self[i:i + 100] + yield self[i] + + def __len__(self): + return len(self._ids) + + def __repr__(self): + return f'{self.__class__.__name__}({self._Model!r}, {self._ids!r})' + + def __setitem__(self, index, value): + if isinstance(index, slice): + size = len(self._ids) + end = min(index.stop or size, size) - 1 + id = map(int, value) + else: + end = index + id = int(value) + self.__check_size(end) + self.__fill(end + 1) + self._ids[index] = id + if isinstance(index, slice): + instance = self.__instantiates(value) + else: + instance = self.__instantiate(value) + super().__setitem__(index, instance) + + def append(self, value): + self.__fill() + self._ids.append(int(value)) + instance = self.__instantiate(value) + super().append(instance) + + def clear(self): + super().clear() + self._ids = [] + + def copy(self): + self.__fill() + return super().copy() + + def count(self, value): + self.__fill() + return super().count(value) + + def extend(self, iterable): + iterable, iterable_ids = tee(iterable) + self.__fill() + self._ids.extend(map(int, iterable_ids)) + super().extend(self.__instantiates(iterable)) + + def index(self, value, start=0, stop=sys.maxsize): + self.__fill(min(stop, len(self._ids)) - 1) + return super().index(value, start, stop) + + def insert(self, index, value): + self.__fill(min(index, len(self._ids))) + self._ids.insert(index, int(value)) + instance = self.__instantiate(value) + super().insert(index, instance) + + def pop(self, index=-1): + if index < 0: + end = len(self.ids) + else: + end = index + 1 + self.__fill(end) + super().pop(index) + self._ids.pop(index) + + def remove(self, value): + if not isinstance(value, self._Model): + raise ValueError(f"unsupported value {value!r}") + self.__fill(self._ids.index(value.id) + 1) + super().remove(value) + self._ids.remove(value.id) + + def reverse(self): + self.__fill() + super().reverse() + self._ids.reverse() + + def sort(self, /, *, key=None, reverse=False): + self.__fill() + super().sort(key=key, reverse=reverse) + self._ids.clear() + self._ids.extend(map(int, list.__iter__(self))) + + class EvalEnvironment(dict): __slots__ = ('_record', '_model') diff --git a/trytond/trytond/tests/test_modelstorage.py b/trytond/trytond/tests/test_modelstorage.py index 8f18dc3831..c6a1eb79f9 100644 --- a/trytond/trytond/tests/test_modelstorage.py +++ b/trytond/trytond/tests/test_modelstorage.py @@ -3,7 +3,7 @@ import warnings -from trytond.model import EvalEnvironment +from trytond.model import BrowseList, EvalEnvironment from trytond.model.exceptions import ( AccessError, DomainValidationError, RequiredValidationError) from trytond.model.modelstorage import _UnsavedRecordError @@ -768,6 +768,301 @@ class ModelStorageTestCase(TestCase): self.assertEqual(notification.description, "Long description") +class BrowseListTestCase(TestCase): + "Test BrowseList" + + @classmethod + def setUpClass(cls): + super().setUpClass() + activate_module('tests') + + @with_transaction() + def test_contains(self): + "Test __contains__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertIn(Model(2), blist) + + @with_transaction() + def test_delitem(self): + "Test __delitem__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + del blist[1] + + self.assertEqual(len(blist), 2) + self.assertEqual(blist.ids, [1, 3]) + self.assertEqual(blist[0]._ids, blist.ids) + + @with_transaction() + def test_eq(self): + "Test __eq__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertEqual(blist, [Model(1), Model(2), Model(3)]) + + @with_transaction() + def test_getitem(self): + "Test __getitem__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertEqual(blist[0], Model(1)) + self.assertEqual(blist[1], Model(2)) + self.assertEqual(blist[2], Model(3)) + with self.assertRaises(IndexError): + blist[42] + + @with_transaction() + def test_getitem_slice(self): + "Test __getitem__ with slice" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertEqual(blist[:0], []) + self.assertEqual(blist[:10], [Model(1), Model(2), Model(3)]) + self.assertEqual(blist[1:3], [Model(2), Model(3)]) + self.assertEqual(blist[10:20], []) + + @with_transaction() + def test_iter(self): + "Test __iter__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertEqual(blist, [Model(i) for i in range(1, 4)]) + + @with_transaction() + def test_len(self): + "Test __len__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertEqual(list.__len__(blist), 0) + self.assertEqual(len(blist), 3) + + list(blist) + self.assertEqual(list.__len__(blist), 3) + self.assertEqual(len(blist), 3) + + @with_transaction() + def test_repr(self): + "Test __repr__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2]) + + self.assertEqual( + repr(blist), + "BrowseList(, [1, 2])") + + @with_transaction() + def test_setitem(self): + "Test __setitem__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + blist[1] = Model(42) + + self.assertEqual(blist[1], Model(42)) + self.assertEqual(blist.ids, [1, 42, 3]) + self.assertEqual(blist[0]._ids, blist.ids) + + @with_transaction() + def test_setitem_slice(self): + "Test __setitem__ slice" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3, 4, 5]) + + blist[1:4] = [Model(42)] + + self.assertEqual(blist, [Model(1), Model(42), Model(5)]) + self.assertEqual(blist.ids, [1, 42, 5]) + + @with_transaction() + def test_not_contains(self): + "Test not __contains__" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertNotIn(Model(42), blist) + + @with_transaction() + def test_append(self): + "Test append" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1]) + blist.append(Model(2)) + + self.assertEqual(blist, [Model(1), Model(2)]) + + @with_transaction() + def test_clear(self): + "Test clear" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + blist.clear() + self.assertEqual(blist, []) + self.assertEqual(blist.ids, []) + + @with_transaction() + def test_copy(self): + "Test copy" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + blist_copy = blist.copy() + self.assertEqual(blist, blist_copy) + self.assertIsInstance(blist_copy, list) + + @with_transaction() + def test_count(self): + "Test count" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3, 1]) + + self.assertEqual(blist.count(Model(1)), 2) + + @with_transaction() + def test_extend(self): + "Test extend" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1]) + blist.extend([Model(2), Model(3)]) + + self.assertEqual(blist, [Model(1), Model(2), Model(3)]) + self.assertEqual(blist.ids, [1, 2, 3]) + + @with_transaction() + def test_index(self): + "Test index" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + + self.assertEqual(blist.index(Model(2)), 1) + + @with_transaction() + def test_index_stop(self): + "Test index with stop" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3, 4, 5]) + + with self.assertRaises(ValueError): + blist.index(Model(4), 0, 2) + self.assertEqual(list.__len__(blist), 2) + + @with_transaction() + def test_insert(self): + "Test insert" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + blist.insert(2, Model(42)) + + self.assertEqual(blist, [Model(1), Model(2), Model(42), Model(3)]) + self.assertEqual(blist.ids, [1, 2, 42, 3]) + + @with_transaction() + def test_pop(self): + "Test pop" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + blist.pop(1) + + self.assertEqual(blist, [Model(1), Model(3)]) + self.assertEqual(blist.ids, [1, 3]) + + @with_transaction() + def test_pop_negative_index(self): + "Test pop negative index" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + blist.pop(-2) + + self.assertEqual(blist, [Model(1), Model(3)]) + self.assertEqual(blist.ids, [1, 3]) + + @with_transaction() + def test_remove(self): + "Test remove" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + blist.remove(Model(2)) + + self.assertEqual(blist, [Model(1), Model(3)]) + self.assertEqual(blist.ids, [1, 3]) + + @with_transaction() + def test_reverse(self): + "Test reverse" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [1, 2, 3]) + blist.reverse() + + self.assertEqual(blist, [Model(3), Model(2), Model(1)]) + self.assertEqual(blist.ids, [3, 2, 1]) + + @with_transaction() + def test_sort(self): + "Test sort" + pool = Pool() + Model = pool.get('test.modelstorage') + + blist = BrowseList(Model, [2, 3, 1]) + blist.sort() + + self.assertEqual(blist, [Model(1), Model(2), Model(3)]) + self.assertEqual(blist.ids, [1, 2, 3]) + + class EvalEnvironmentTestCase(TestCase): "Test EvalEnvironment"