From 7892f8ae56333eed1de05fae7112f707402ddfb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Tue, 11 Oct 2022 00:44:51 +0200 Subject: [PATCH] Use declarative index definition for ModelSQL issue5757 review361251002 --- CHANGELOG | 1 + doc/ref/backend.rst | 60 ++++++- doc/ref/fields.rst | 4 - doc/ref/models.rst | 49 +++++- trytond/backend/database.py | 3 + trytond/backend/postgresql/database.py | 41 ++++- trytond/backend/postgresql/table.py | 230 ++++++++++++++++++------- trytond/backend/sqlite/table.py | 140 +++++++++------ trytond/backend/table.py | 50 +++++- trytond/ir/action.py | 22 +-- trytond/ir/cron.py | 7 +- trytond/ir/email_.py | 4 +- trytond/ir/error.py | 11 +- trytond/ir/export.py | 2 +- trytond/ir/model.py | 62 +++---- trytond/ir/module.py | 4 +- trytond/ir/queue_.py | 23 ++- trytond/ir/resource.py | 12 +- trytond/ir/rule.py | 29 +++- trytond/ir/session.py | 24 ++- trytond/ir/translation.py | 38 +++- trytond/ir/trigger.py | 25 +-- trytond/ir/ui/icon.py | 2 +- trytond/ir/ui/menu.py | 5 +- trytond/ir/ui/view.py | 60 ++++--- trytond/model/__init__.py | 4 +- trytond/model/fields/binary.py | 4 +- trytond/model/fields/boolean.py | 4 +- trytond/model/fields/char.py | 6 +- trytond/model/fields/date.py | 12 +- trytond/model/fields/dict.py | 5 +- trytond/model/fields/field.py | 4 +- trytond/model/fields/float.py | 4 +- trytond/model/fields/many2one.py | 4 +- trytond/model/fields/multiselection.py | 4 +- trytond/model/fields/reference.py | 4 +- trytond/model/fields/selection.py | 4 +- trytond/model/modelsql.py | 141 ++++++++++++++- trytond/model/order.py | 9 +- trytond/modules/__init__.py | 19 +- trytond/res/__init__.py | 3 + trytond/res/group.py | 2 +- trytond/res/ir.py | 83 +++++++-- trytond/res/user.py | 38 ++-- trytond/tests/mptt.py | 7 +- trytond/tests/path.py | 2 +- trytond/tests/test_model_index.py | 140 +++++++++++++++ 47 files changed, 1062 insertions(+), 349 deletions(-) create mode 100644 trytond/tests/test_model_index.py diff --git a/CHANGELOG b/CHANGELOG index e7ce36afda..9ad08464eb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,4 @@ +* Use declarative index definition for ModelSQL * Add configuration entries for unaccent and similarity functions * Add authentication services * Schedule actions using server timezone diff --git a/doc/ref/backend.rst b/doc/ref/backend.rst index a9c0216020..691aa951ae 100644 --- a/doc/ref/backend.rst +++ b/doc/ref/backend.rst @@ -7,10 +7,18 @@ Backend The backend manages the database connection and schema manipulation. +.. contents:: + :local: + :backlinks: entry + :depth: 1 + .. attribute:: name The name of the backend configured. +Database +======== + .. class:: Database(name) Manage the connection to the named database. @@ -109,6 +117,10 @@ The backend manages the database connection and schema manipulation. Return if the database suppport unaccentuated function. +.. method:: Database.has_unaccent_indexable() + + Return if the database suppport unaccentuated function in index. + .. method:: Database.unaccent(value) Return the SQL expression of unaccentuated ``value``. @@ -211,6 +223,9 @@ The backend manages the database connection and schema manipulation. Rteurn the SQL expression to test if the JSON ``column`` contains ``json``. +TableHandler +============ + .. class:: TableHandler(model[, history]) Handle table for the ``model``. @@ -220,6 +235,10 @@ The backend manages the database connection and schema manipulation. The maximing length of named data for the database. +.. attribute:: TableHandler.index_translators + + Contain the :class:`IndexTranslator` for the database. + .. classmethod:: TableHandler.table_exist(table_name) Return if the named table exists. @@ -271,12 +290,6 @@ The backend manages the database connection and schema manipulation. Drop the foreign key constrant on the named column. ``table`` can be used to alter another table. -.. method:: TableHandler.index_action(columns[, action[, where[, table]]]) - - Add or remove index on listed ``columns``. - ``where`` is an optional SQL expression for partial index. - ``table`` can be used to alter another table. - .. method:: TableHandler.not_null_action(column_name[, action]) Add or remove ``NOT NULL`` on the named column. @@ -291,6 +304,11 @@ The backend manages the database connection and schema manipulation. Drop the named ``ident`` constraint. ``table`` can be used to alter another table. +.. method:: TableHandler.set_indexes(indexes) + + Create the :class:`indexes ` if possible and drop + others having ``idx_`` as prefix or ``_index`` as suffix. + .. method:: TableHandler.drop_column(column_name) Drop the named column. @@ -301,9 +319,35 @@ The backend manages the database connection and schema manipulation. ``model``. Set ``cascade`` to drop objects that depend on the table. -.. classmethod:: TableHandler.convert_name(name) +.. classmethod:: TableHandler.convert_name(name[, reserved]) - Convert the data name to be lower than namedatalen. + Convert the data name to be lower than namedatalen minus reserved. + +.. method:: Database.index_translator_for(index) + + Return the best :class:`IndexTranslator` for the given index. + +IndexTranslator +=============== + +.. class:: IndexTranslator + + Convert an :class:`~trytond.model.Index` into a concrete index. + +.. classmethod:: IndexTranslator.definition(index) + + Return the name, SQL query and parameters to create the :class:`index + `. + +.. classmethod:: IndexTranslator.score(index) + + Return a score related to the fitness of using this translator for the + :class:`index `. + A score of ``0`` means that the translator is unsuited for the requested + usage. + +Exceptions +========== .. exception:: DatabaseIntegrityError diff --git a/doc/ref/fields.rst b/doc/ref/fields.rst index 969d36ab75..b2c1a9fe07 100644 --- a/doc/ref/fields.rst +++ b/doc/ref/fields.rst @@ -63,10 +63,6 @@ All are optional except :attr:`~Field.string`. The values are :class:`~trytond.pyson.PYSON` statements that is evaluated with the values of the record. -.. attribute:: Field.select - - If ``True``, the content of the field is indexed. - .. attribute:: Field.on_change A set of field names. diff --git a/doc/ref/models.rst b/doc/ref/models.rst index 9c823b91da..0b77c55de9 100644 --- a/doc/ref/models.rst +++ b/doc/ref/models.rst @@ -8,7 +8,7 @@ Models .. contents:: :local: :backlinks: entry - :depth: 1 + :depth: 2 Model ===== @@ -527,6 +527,11 @@ Class attributes are: xml id The message id for :meth:`~trytond.i18n.gettext` +.. attribute:: ModelSQL._sql_indexes + + A :py:class:`set ` containing the :class:`Index` that are created on + the table. + Class methods: .. classmethod:: ModelSQL.__table__() @@ -680,6 +685,48 @@ Instance attributes: The clause for which the exclusion applies. +Index +----- + +.. class:: Index(table[, \*expressions, [, \*\*options]]) + +Represent a SQL index for the ``table`` for the sequence of ``expressions``. +An ``expression`` is a :py:class:`tuple ` of SQL expression and a +:attr:`~Index.Usage`. + +Available options are: + + * ``include``: a list of columns to include in the index + * ``where``: the where clause for partial index + +.. attribute:: Index.Unaccent(expression) + +Apply unaccent function if the database supports it. + +.. attribute:: Index.Usage(\*\*options) + +Represent a usage of a SQL expression. +Available options are: + + * ``collation``: the name of the collation + * ``order``: the sort order + +.. attribute:: Index.Equality(\*\*options) + +Represent an equality usage. + +.. attribute:: Index.Range(\*\*options) + +Represent an range usage. + +.. attribute:: Index.Similarity(\*\*options) + +Represent a similar usage only for text. +Additional options are available: + + * ``begin``: optimize for constant pattern and anchored to the beginning of + the string + Workflow ======== diff --git a/trytond/backend/database.py b/trytond/backend/database.py index d363720975..22c7ec9a52 100644 --- a/trytond/backend/database.py +++ b/trytond/backend/database.py @@ -92,6 +92,9 @@ class DatabaseInterface(object): def has_unaccent(self): return False + def has_unaccent_indexable(self): + return False + def unaccent(self, value): return value diff --git a/trytond/backend/postgresql/database.py b/trytond/backend/postgresql/database.py index d7b2323229..0537cddc28 100644 --- a/trytond/backend/postgresql/database.py +++ b/trytond/backend/postgresql/database.py @@ -181,6 +181,7 @@ class JSONContains(BinaryOperator): class Database(DatabaseInterface): + index_translators = [] _lock = RLock() _databases = defaultdict(dict) _connpool = None @@ -190,7 +191,8 @@ class Database(DatabaseInterface): _current_user = None _has_returning = None _has_select_for_skip_locked = None - _has_proc = defaultdict(dict) + _has_proc = defaultdict(lambda: defaultdict(dict)) + _extensions = defaultdict(dict) _search_full_text_languages = defaultdict(dict) flavor = Flavor(ilike=True) @@ -515,25 +517,31 @@ class Database(DatabaseInterface): def has_sequence(cls): return True - def has_proc(self, name): - if name in self._has_proc[self.name]: - return self._has_proc[self.name][name] + def has_proc(self, name, property='oid'): + if (name in self._has_proc[self.name] + and property in self._has_proc[self.name][name]): + return self._has_proc[self.name][name][property] connection = self.get_connection() result = False try: cursor = connection.cursor() cursor.execute( - "SELECT 1 FROM pg_proc WHERE proname=%s", - (name,)) - result = bool(cursor.rowcount) + SQL('SELECT {} FROM pg_proc WHERE proname=%s').format( + Identifier(property)), (name,)) + result = cursor.fetchone() + if result: + result, = result finally: self.put_connection(connection) - self._has_proc[self.name][name] = result + self._has_proc[self.name][name][property] = result return result def has_unaccent(self): return self.has_proc(Unaccent._function) + def has_unaccent_indexable(self): + return self.has_proc(Unaccent._function, 'provolatile') == 'i' + def has_similarity(self): return self.has_proc(Similarity._function) @@ -715,6 +723,23 @@ class Database(DatabaseInterface): def has_channel(self): return True + def has_extension(self, extension_name): + if extension_name in self._extensions[self.name]: + return self._extensions[self.name][extension_name] + + connection = self.get_connection() + result = False + try: + cursor = connection.cursor() + cursor.execute( + "SELECT 1 FROM pg_extension WHERE extname=%s", + (extension_name,)) + result = bool(cursor.rowcount) + finally: + self.put_connection(connection) + self._extensions[self.name][extension_name] = result + return result + def json_get(self, column, key=None): column = Cast(column, 'jsonb') if key: diff --git a/trytond/backend/postgresql/table.py b/trytond/backend/postgresql/table.py index cb178d707c..8524179d59 100644 --- a/trytond/backend/postgresql/table.py +++ b/trytond/backend/postgresql/table.py @@ -3,9 +3,10 @@ import logging import re -from psycopg2.sql import SQL, Composed, Identifier +from psycopg2.sql import SQL, Identifier -from trytond.backend.table import TableHandlerInterface +from trytond.backend.table import ( + IndexTranslatorInterface, TableHandlerInterface) from trytond.transaction import Transaction __all__ = ['TableHandler'] @@ -16,6 +17,7 @@ VARCHAR_SIZE_RE = re.compile(r'VARCHAR\(([0-9]+)\)') class TableHandler(TableHandlerInterface): namedatalen = 64 + index_translators = [] def _init(self, model, history=False): super()._init(model, history=history) @@ -212,22 +214,18 @@ class TableHandler(TableHandlerInterface): "JOIN pg_class cl on (cl.oid = ind.indrelid) " "JOIN pg_namespace n ON (cl.relnamespace = n.oid) " "JOIN pg_class cl2 on (cl2.oid = ind.indexrelid) " - "WHERE cl.relname = %s AND n.nspname = %s", + "WHERE cl.relname = %s AND n.nspname = %s " + "AND NOT ind.indisprimary AND NOT ind.indisunique", (self.table_name, self.table_schema)) self.__indexes = [l[0] for l in cursor] return self.__indexes - def _update_definitions(self, - columns=None, constraints=None, indexes=None): - if columns is None and constraints is None and indexes is None: - columns = constraints = indexes = True + def _update_definitions(self, columns=True, constraints=True): if columns: self.__columns = None if constraints: self.__constraints = None self.__fk_deltypes = None - if indexes: - self.__indexes = None def alter_size(self, column_name, column_type): cursor = Transaction().connection.cursor() @@ -396,58 +394,6 @@ class TableHandler(TableHandlerInterface): def drop_fk(self, column_name, table=None): self.drop_constraint(column_name + '_fkey', table=table) - def index_action(self, columns, action='add', where='', table=None): - if isinstance(columns, str): - columns = [columns] - - def stringify(column): - if isinstance(column, str): - return column - else: - return ('_'.join( - map(str, (column,) + column.params)) - .replace('"', '') - .replace('%s', '__')) - - name = [table or self.table_name] - name.append('_'.join(map(stringify, columns))) - if where: - name.append('+where') - name.append(stringify(where)) - name.append('index') - index_name = self.convert_name('_'.join(name)) - - with Transaction().connection.cursor() as cursor: - if action == 'add': - if index_name in self._indexes: - return - params = sum( - (c.params for c in columns if hasattr(c, 'params')), ()) - if where: - params += where.params - where = ' WHERE %s' % where - cursor.execute(Composed([ - SQL('CREATE INDEX {name} ON {table} ({columns})') - .format( - name=Identifier(index_name), - table=Identifier(self.table_name), - columns=SQL(',').join( - (Identifier(c) if isinstance(c, str) - else SQL(str(c))) - for c in columns - )), - SQL(where) - ]), - params) - self._update_definitions(indexes=True) - elif action == 'remove': - if index_name in self._indexes: - cursor.execute(SQL('DROP INDEX {}').format( - Identifier(index_name))) - self._update_definitions(indexes=True) - else: - raise Exception('Index action not supported!') - def not_null_action(self, column_name, action='add'): if not self.column_exist(column_name): return @@ -511,6 +457,27 @@ class TableHandler(TableHandlerInterface): Identifier(self.table_name), Identifier(ident))) self._update_definitions(constraints=True) + def set_indexes(self, indexes): + cursor = Transaction().connection.cursor() + old = set(self._indexes) + for index in indexes: + translator = self.index_translator_for(index) + if translator: + name, query, params = translator.definition(index) + name = '_'.join([self.table_name, name]) + name = 'idx_' + self.convert_name(name, reserved=len('idx_')) + cursor.execute( + SQL('CREATE INDEX IF NOT EXISTS {} ON {} USING {}').format( + Identifier(name), + Identifier(self.table_name), + query), + params) + old.discard(name) + for name in old: + if name.startswith('idx_') or name.endswith('_index'): + cursor.execute(SQL('DROP INDEX {}').format(Identifier(name))) + self.__indexes = None + def drop_column(self, column_name): if not self.column_exist(column_name): return @@ -529,3 +496,144 @@ class TableHandler(TableHandlerInterface): if cascade: query = query + ' CASCADE' cursor.execute(SQL(query).format(Identifier(table))) + + +class IndexMixin: + + _type = None + + def __init_subclass__(cls): + TableHandler.index_translators.append(cls) + + @classmethod + def definition(cls, index): + expr_template = SQL('{expression} {collate} {opclass} {order}') + indexed_expressions = cls._get_indexed_expressions(index) + expressions = [] + params = [] + for expression, usage in indexed_expressions: + expressions.append(expr_template.format( + **cls._get_expression_variables(expression, usage))) + params.extend(expression.params) + + include = SQL('') + if index.options.get('include'): + include = SQL('INCLUDE ({columns})').format( + columns=SQL(',').join(map( + lambda c: SQL(str(c)), + index.options.get('include')))) + + where = SQL('') + if index.options.get('where'): + where = SQL('WHERE {where}').format( + where=SQL(str(index.options['where']))) + params.extend(index.options['where'].params) + + query = SQL('{type} ({expressions}) {include} {where}').format( + type=SQL(cls._type), + expressions=SQL(',').join(expressions), + include=include, + where=where) + name = cls._get_name(query, params) + return name, query, params + + @classmethod + def _get_indexed_expressions(cls, index): + return index.expressions + + @classmethod + def _get_expression_variables(cls, expression, usage): + variables = { + 'expression': SQL(str(expression)), + 'collate': SQL(''), + 'opclass': SQL(''), + 'order': SQL(''), + } + if usage.options.get('collation'): + variables['collate'] = SQL('COLLATE {}').format( + usage.options['collation']) + if usage.options.get('order'): + order = usage.options['order'].upper() + variables['order'] = SQL(order) + return variables + + +class HashTranslator(IndexMixin, IndexTranslatorInterface): + _type = 'HASH' + + @classmethod + def score(cls, index): + if (len(index.expressions) > 1 + or index.expressions[0][1].__class__.__name__ != 'Equality'): + return 0 + return 100 + + @classmethod + def _get_indexed_expressions(cls, index): + return [ + (e, u) for e, u in index.expressions + if u.__class__.__name__ == 'Equality'][:1] + + +class BTreeTranslator(IndexMixin, IndexTranslatorInterface): + _type = 'BTREE' + + @classmethod + def score(cls, index): + score = 0 + for _, usage in index.expressions: + if usage.__class__.__name__ == 'Range': + score += 100 + elif usage.__class__.__name__ == 'Equality': + score += 50 + elif usage.__class__.__name__ == 'Similarity': + score += 20 + if usage.options.get('begin'): + score += 100 + return score + + @classmethod + def _get_expressions(cls, index): + return [ + (e, u) for e, u in index.expressions + if u.__class__.__name__ in {'Equality', 'Range'}] + + @classmethod + def _get_expression_variables(cls, expression, usage): + params = super()._get_expression_variables(expression, usage) + if (usage.__class__.__name__ == 'Similarity' + and not usage.options.get('collation')): + # text_pattern_ops and varchar_pattern_ops are the same + params['opclass'] = SQL('varchar_pattern_ops') + return params + + +class TrigramTranslator(IndexMixin, IndexTranslatorInterface): + _type = 'GIN' + + @classmethod + def score(cls, index): + has_trigram = Transaction().database.has_extension('pg_trgm') + if not has_trigram: + return 0 + + score = 0 + for _, usage in index.expressions: + if usage.__class__.__name__ == 'Similarity': + score += 100 + else: + return 0 + return score + + @classmethod + def _get_expressions(cls, index): + return [ + (e, u) for e, u in index.expressions + if u.__class__.__name__ == 'Similarity'] + + @classmethod + def _get_expression_variables(cls, expression, usage): + params = super()._get_expression_variables(expression, usage) + if usage.__class__.__name__ == 'Similarity': + params['opclass'] = SQL('gin_trgm_ops') + return params diff --git a/trytond/backend/sqlite/table.py b/trytond/backend/sqlite/table.py index 30ba2feda6..5701a33a44 100644 --- a/trytond/backend/sqlite/table.py +++ b/trytond/backend/sqlite/table.py @@ -6,7 +6,8 @@ import re import warnings from weakref import WeakKeyDictionary -from trytond.backend.table import TableHandlerInterface +from trytond.backend.table import ( + IndexTranslatorInterface, TableHandlerInterface) from trytond.transaction import Transaction from .database import sqlite @@ -23,6 +24,7 @@ def _escape_identifier(name): class TableHandler(TableHandlerInterface): __handlers = WeakKeyDictionary() + index_translators = [] def _init(self, model, history=False): super()._init(model, history=history) @@ -184,13 +186,9 @@ class TableHandler(TableHandlerInterface): self.__indexes = [l[1] for l in cursor] return self.__indexes - def _update_definitions(self, columns=None, indexes=None): - if columns is None and indexes is None: - columns = indexes = True + def _update_definitions(self, columns=True): if columns: self.__columns = None - if indexes: - self.__indexes = None def alter_size(self, column_name, column_type): self._recreate_table({column_name: {'size': column_type}}) @@ -288,56 +286,6 @@ class TableHandler(TableHandlerInterface): def drop_fk(self, column_name, table=None): warnings.warn('Unable to drop foreign key with SQLite backend') - def index_action(self, columns, action='add', where='', table=None): - if isinstance(columns, str): - columns = [columns] - - def stringify(column): - if isinstance(column, str): - return column - else: - return ('_'.join( - map(str, (column,) + column.params)) - .replace('?', '__')) - - name = [table or self.table_name] - name.append('_'.join(map(stringify, columns))) - if where: - name.append('+where') - name.append(stringify(where)) - name.append('index') - index_name = self.convert_name('_'.join(name)) - - cursor = Transaction().connection.cursor() - if action == 'add': - if index_name in self._indexes: - return - params = sum( - (c.params for c in columns if hasattr(c, 'params')), ()) - if where: - params += where.params - where = ' WHERE %s' % where - if params: - warnings.warn('Unable to create index with parameters') - return - cursor.execute( - 'CREATE INDEX %s ON %s (%s)' % ( - _escape_identifier(index_name), - _escape_identifier(self.table_name), - ','.join( - _escape_identifier(c) if isinstance(c, str) else str(c) - for c in columns) - ) + where, - params) - self._update_definitions(indexes=True) - elif action == 'remove': - if index_name in self._indexes: - cursor.execute('DROP INDEX %s' - % (_escape_identifier(index_name),)) - self._update_definitions(indexes=True) - else: - raise Exception('Index action not supported!') - def not_null_action(self, column_name, action='add'): if not self.column_exist(column_name): return @@ -355,6 +303,31 @@ class TableHandler(TableHandlerInterface): def drop_constraint(self, ident, table=None): warnings.warn('Unable to drop constraint with SQLite backend') + def set_indexes(self, indexes): + cursor = Transaction().connection.cursor() + old = set(self._indexes) + for index in indexes: + translator = self.index_translator_for(index) + if translator: + name, query, params = translator.definition(index) + name = '_'.join([self.table_name, name]) + name = 'idx_' + self.convert_name(name, reserved=len('idx_')) + # SQLite does not support parameters for index creation + if not params: + cursor.execute( + 'CREATE INDEX IF NOT EXISTS %s ON %s %s' % ( + _escape_identifier(name), + _escape_identifier(self.table_name), + query), + params) + else: + warnings.warn("Can not create index with parameters") + old.discard(name) + for name in old: + if name.startswith('idx_') or name.endswith('_index'): + cursor.execute('DROP INDEX %s' % _escape_identifier(name)) + self.__indexes = None + def drop_column(self, column_name): if not self.column_exist(column_name): return @@ -378,3 +351,58 @@ class TableHandler(TableHandlerInterface): if cascade: query = query + ' CASCADE' cursor.execute(query) + + +class IndexMixin: + + def __init_subclass__(cls): + TableHandler.index_translators.append(cls) + + @classmethod + def definition(cls, index): + expr_template = '%(expression)s %(collate)s %(order)s' + params = [] + expressions = [] + for expression, usage in index.expressions: + expressions.append(expr_template % + cls._get_expression_variables(expression, usage)) + params.extend(expression.params) + + where = '' + if index.options.get('where'): + where = 'WHERE %s' % index.options['where'] + params.extend(index.options['where'].params) + + query = '(%(expressions)s) %(where)s' % { + 'expressions': ','.join(expressions), + 'where': where, + } + name = cls._get_name(query, params) + return name, query, params + + @classmethod + def _get_expression_variables(cls, expression, usage): + variables = { + 'expression': str(expression), + 'collate': '', + 'order': '', + } + if usage.options.get('collation'): + variables['collate'] = 'COLLATE %s' % usage.options['collation'] + if usage.options.get('order'): + order = usage.options['order'].upper() + for predicate in ['NULLS FIRST', 'NULLS LAST']: + if order.endswith(predicate): + order = order[:-len(predicate)] + variables['order'] = order + return variables + + +class IndexTranslator(IndexMixin, IndexTranslatorInterface): + + @classmethod + def score(cls, index): + supported_indexes_count = sum( + int(u.__class__.__name__ in {'Equality', 'Range'}) + for _, u in index.expressions) + return supported_indexes_count * 100 diff --git a/trytond/backend/table.py b/trytond/backend/table.py index 0525b5f42f..58e4f86d70 100644 --- a/trytond/backend/table.py +++ b/trytond/backend/table.py @@ -11,6 +11,7 @@ class TableHandlerInterface(object): Define generic interface to handle database table ''' namedatalen = None + index_translators = None __handlers = WeakKeyDictionary() def __new__(cls, model, history=False): @@ -75,9 +76,6 @@ class TableHandlerInterface(object): def drop_fk(self, column_name, table=None): raise NotImplementedError - def index_action(self, columns, action='add', where=None, table=None): - raise NotImplementedError - def not_null_action(self, column_name, action='add'): raise NotImplementedError @@ -87,6 +85,9 @@ class TableHandlerInterface(object): def drop_constraint(self, ident, table=None): raise NotImplementedError + def create_index(self, index): + raise NotImplementedError + def drop_column(self, column_name): raise NotImplementedError @@ -95,9 +96,42 @@ class TableHandlerInterface(object): raise NotImplementedError @classmethod - def convert_name(cls, name): - if cls.namedatalen and len(name) >= cls.namedatalen: - if isinstance(name, str): - name = name.encode('utf-8') - name = hashlib.sha256(name).hexdigest()[:cls.namedatalen - 1] + def convert_name(cls, name, reserved=0): + if cls.namedatalen: + length = cls.namedatalen - reserved + if length <= 0: + raise ValueError + if len(name) >= length: + if isinstance(name, str): + name = name.encode('utf-8') + name = hashlib.sha256(name).hexdigest()[:length - 1] return name + + def index_translator_for(self, index): + return next( + filter( + lambda t: t.score(index) > 0, + sorted( + self.index_translators, key=lambda t: t.score(index), + reverse=True)), + None) + + +class IndexTranslatorInterface: + + @classmethod + def _get_name(cls, query, params): + def hash_(s): + return hashlib.shake_128(s.encode('utf-8')).hexdigest(16) + names = [str(query)] + if params: + names.append(str(params)) + return '_'.join(map(hash_, names)) + + @classmethod + def definition(cls, index): + raise NotImplementedError + + @classmethod + def score(cls, index): + raise NotImplementedError diff --git a/trytond/ir/action.py b/trytond/ir/action.py index c16cb4ebc5..ec243422c2 100644 --- a/trytond/ir/action.py +++ b/trytond/ir/action.py @@ -12,7 +12,7 @@ from trytond.cache import Cache, MemoryCache from trytond.config import config from trytond.i18n import gettext from trytond.model import ( - DeactivableMixin, ModelSingleton, ModelSQL, ModelStorage, ModelView, + DeactivableMixin, Index, ModelSingleton, ModelSQL, ModelStorage, ModelView, fields, sequence_ordered) from trytond.model.exceptions import ValidationError from trytond.pool import Pool @@ -152,22 +152,22 @@ class ActionKeyword(ModelSQL, ModelView): ], string='Keyword', required=True) model = fields.Reference('Model', selection='models_get') action = fields.Many2One('ir.action', 'Action', - ondelete='CASCADE', select=True) + ondelete='CASCADE') _get_keyword_cache = Cache('ir_action_keyword.get_keyword') @classmethod def __setup__(cls): super(ActionKeyword, cls).__setup__() + table = cls.__table__() + cls.__rpc__.update({ 'get_keyword': RPC(cache=dict(days=1)), }) - - @classmethod - def __register__(cls, module_name): - super(ActionKeyword, cls).__register__(module_name) - - table = cls.__table_handler__(module_name) - table.index_action(['keyword', 'model'], 'add') + cls._sql_indexes.add( + Index( + table, + (table.keyword, Index.Equality()), + (table.model, Index.Equality()))) @classmethod def validate_fields(cls, actions, field_names): @@ -561,7 +561,7 @@ class ActionReport(ActionMixin, ModelSQL, ModelView): "Record Name", translate=True, help="A Genshi expression to compute the name using 'record'.\n" "Leave empty for the default name.",) - module = fields.Char('Module', readonly=True, select=True) + module = fields.Char('Module', readonly=True) _template_cache = MemoryCache('ir.action.report.template', context=False) @classmethod @@ -997,7 +997,7 @@ class ActionActWindowDomain( domain = fields.Char('Domain') count = fields.Boolean('Count') act_window = fields.Many2One('ir.action.act_window', 'Action', - select=True, required=True, ondelete='CASCADE') + required=True, ondelete='CASCADE') @classmethod def __register__(cls, module_name): diff --git a/trytond/ir/cron.py b/trytond/ir/cron.py index ab59738923..1ca3e0e139 100644 --- a/trytond/ir/cron.py +++ b/trytond/ir/cron.py @@ -10,7 +10,7 @@ from trytond import backend from trytond.config import config from trytond.exceptions import UserError, UserWarning from trytond.model import ( - DeactivableMixin, ModelSQL, ModelView, dualmethod, fields) + DeactivableMixin, Index, ModelSQL, ModelView, dualmethod, fields) from trytond.pool import Pool from trytond.pyson import Eval from trytond.status import processing @@ -69,7 +69,7 @@ class Cron(DeactivableMixin, ModelSQL, ModelView): depends=['interval_type']) timezone = fields.Function(fields.Char("Timezone"), 'get_timezone') - next_call = fields.DateTime("Next Call", select=True) + next_call = fields.DateTime("Next Call") method = fields.Selection([ ('ir.trigger|trigger_time', "Run On Time Triggers"), ('ir.queue|clean', "Clean Task Queue"), @@ -79,11 +79,14 @@ class Cron(DeactivableMixin, ModelSQL, ModelView): @classmethod def __setup__(cls): super(Cron, cls).__setup__() + table = cls.__table__() + cls._buttons.update({ 'run_once': { 'icon': 'tryton-launch', }, }) + cls._sql_indexes.add(Index(table, (table.next_call, Index.Range()))) @classmethod def __register__(cls, module_name): diff --git a/trytond/ir/email_.py b/trytond/ir/email_.py index dc3653a5f7..84debecf3f 100644 --- a/trytond/ir/email_.py +++ b/trytond/ir/email_.py @@ -245,8 +245,8 @@ class EmailAddress(ModelSQL): __name__ = 'ir.email.address' email = fields.Many2One( - 'ir.email', "E-mail", required=True, ondelete='CASCADE', select=True) - address = fields.Char("Address", required=True, select=True) + 'ir.email', "E-mail", required=True, ondelete='CASCADE') + address = fields.Char("Address", required=True) class EmailTemplate(ModelSQL, ModelView): diff --git a/trytond/ir/error.py b/trytond/ir/error.py index fb1cab69da..93e7161b9c 100644 --- a/trytond/ir/error.py +++ b/trytond/ir/error.py @@ -6,7 +6,7 @@ import logging from trytond.config import config from trytond.exceptions import UserError, UserWarning -from trytond.model import ModelSQL, ModelView, Workflow, fields +from trytond.model import Index, ModelSQL, ModelView, Workflow, fields from trytond.pool import Pool from trytond.pyson import Eval from trytond.tools import firstline @@ -72,11 +72,18 @@ class Error(Workflow, ModelView, ModelSQL): ('open', "Open"), ('processing', "Processing"), ('solved', "Solved"), - ], "State", readonly=True, select=True, sort=False) + ], "State", readonly=True, sort=False) @classmethod def __setup__(cls): super().__setup__() + table = cls.__table__() + + cls._sql_indexes.add( + Index( + table, + (table.state, Index.Equality()), + where=table.state.in_(['open', 'processing']))) cls._transitions |= { ('open', 'processing'), ('processing', 'solved'), diff --git a/trytond/ir/export.py b/trytond/ir/export.py index 01a2171a94..17d4da6f76 100644 --- a/trytond/ir/export.py +++ b/trytond/ir/export.py @@ -61,5 +61,5 @@ class ExportLine(_ClearCache, ModelSQL, ModelView): "Export line" __name__ = 'ir.export.line' name = fields.Char('Name') - export = fields.Many2One('ir.export', 'Export', select=True, required=True, + export = fields.Many2One('ir.export', 'Export', required=True, ondelete='CASCADE') diff --git a/trytond/ir/model.py b/trytond/ir/model.py index 6304018f24..3ff644f51e 100644 --- a/trytond/ir/model.py +++ b/trytond/ir/model.py @@ -15,8 +15,8 @@ from sql.operators import Equal from trytond.cache import Cache from trytond.i18n import gettext from trytond.model import ( - DeactivableMixin, EvalEnvironment, Exclude, ModelSingleton, ModelSQL, - ModelView, Unique, Workflow, fields) + DeactivableMixin, EvalEnvironment, Exclude, Index, ModelSingleton, + ModelSQL, ModelView, Unique, Workflow, fields) from trytond.model.exceptions import AccessError, ValidationError from trytond.pool import Pool from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder @@ -231,7 +231,7 @@ class ModelField(ModelSQL, ModelView): }, depends=['module']) model = fields.Many2One('ir.model', 'Model', required=True, - select=True, ondelete='CASCADE', + ondelete='CASCADE', states={ 'readonly': Bool(Eval('module')), }, @@ -859,10 +859,7 @@ class ModelButton(DeactivableMixin, ModelSQL, ModelView): confirm = fields.Text("Confirm", translate=True, help="Text to ask user confirmation when clicking the button.") model = fields.Many2One('ir.model', 'Model', required=True, readonly=True, - ondelete='CASCADE', select=True) - groups = fields.Many2Many('ir.model.button-res.group', 'button', 'group', - 'Groups') - _groups_cache = Cache('ir.model.button.groups') + ondelete='CASCADE') rules = fields.One2Many('ir.model.button.rule', 'button', "Rules") _rules_cache = Cache('ir.model.button.rules') clicks = fields.One2Many('ir.model.button.click', 'button', "Clicks") @@ -909,8 +906,6 @@ class ModelButton(DeactivableMixin, ModelSQL, ModelView): @classmethod def create(cls, vlist): result = super(ModelButton, cls).create(vlist) - # Restart the cache for get_groups and get_rules - cls._groups_cache.clear() cls._rules_cache.clear() cls._reset_cache.clear() cls._view_attributes_cache.clear() @@ -919,8 +914,6 @@ class ModelButton(DeactivableMixin, ModelSQL, ModelView): @classmethod def write(cls, buttons, values, *args): super(ModelButton, cls).write(buttons, values, *args) - # Restart the cache for get_groups and get_rules - cls._groups_cache.clear() cls._rules_cache.clear() cls._reset_cache.clear() cls._view_attributes_cache.clear() @@ -929,7 +922,6 @@ class ModelButton(DeactivableMixin, ModelSQL, ModelView): def delete(cls, buttons): super(ModelButton, cls).delete(buttons) # Restart the cache for get_groups and get_rules - cls._groups_cache.clear() cls._rules_cache.clear() cls._reset_cache.clear() cls._view_attributes_cache.clear() @@ -943,27 +935,6 @@ class ModelButton(DeactivableMixin, ModelSQL, ModelView): default.setdefault('clicks') return super(ModelButton, cls).copy(buttons, default=default) - @classmethod - def get_groups(cls, model, name): - ''' - Return a set of group ids for the named button on the model. - ''' - key = (model, name) - groups = cls._groups_cache.get(key) - if groups is not None: - return groups - buttons = cls.search([ - ('model.model', '=', model), - ('name', '=', name), - ]) - if not buttons: - groups = set() - else: - button, = buttons - groups = set(g.id for g in button.groups) - cls._groups_cache.set(key, groups) - return groups - @classmethod def get_rules(cls, model, name): 'Return a list of rules to apply on the named button of the model' @@ -1176,22 +1147,21 @@ class ModelButtonReset(ModelSQL): __name__ = 'ir.model.button-button.reset' button_ruled = fields.Many2One( 'ir.model.button', "Button Ruled", - required=True, ondelete='CASCADE', select=True) + required=True, ondelete='CASCADE') button = fields.Many2One( 'ir.model.button', "Button", - required=True, ondelete='CASCADE', select=True) + required=True, ondelete='CASCADE') class ModelData(ModelSQL, ModelView): "Model data" __name__ = 'ir.model.data' fs_id = fields.Char('Identifier on File System', required=True, - help="The id of the record as known on the file system.", - select=True) - model = fields.Char('Model', required=True, select=True) - module = fields.Char('Module', required=True, select=True) + help="The id of the record as known on the file system.") + model = fields.Char('Model', required=True) + module = fields.Char('Module', required=True) db_id = fields.Integer( - "Resource ID", select=True, + "Resource ID", states={ 'required': ~Eval('noupdate', False), }, @@ -1212,7 +1182,17 @@ class ModelData(ModelSQL, ModelView): ('fs_id_module_model_uniq', Unique(table, table.fs_id, table.module, table.model), 'The triple (fs_id, module, model) must be unique!'), - ] + ] + cls._sql_indexes.update({ + Index( + table, + (table.fs_id, Index.Equality()), + (table.module, Index.Equality()), + (table.model, Index.Equality())), + Index( + table, + (table.module, Index.Equality())), + }) cls._buttons.update({ 'sync': { 'invisible': ~Eval('out_of_sync'), diff --git a/trytond/ir/module.py b/trytond/ir/module.py index 25430cdb34..bb727b2bd0 100644 --- a/trytond/ir/module.py +++ b/trytond/ir/module.py @@ -331,7 +331,7 @@ class ModuleDependency(ModelSQL, ModelView): "Module dependency" __name__ = "ir.module.dependency" name = fields.Char('Name') - module = fields.Many2One('ir.module', 'Module', select=True, + module = fields.Many2One('ir.module', 'Module', ondelete='CASCADE', required=True) state = fields.Function(fields.Selection([ ('not activated', 'Not Activated'), @@ -380,7 +380,7 @@ class ModuleConfigWizardItem(sequence_ordered(), ModelSQL, ModelView): state = fields.Selection([ ('open', 'Open'), ('done', 'Done'), - ], string="State", required=True, select=True, sort=False) + ], string="State", required=True, sort=False) @classmethod def __register__(cls, module_name): diff --git a/trytond/ir/queue_.py b/trytond/ir/queue_.py index 6b963c00a3..86cd658982 100644 --- a/trytond/ir/queue_.py +++ b/trytond/ir/queue_.py @@ -7,7 +7,7 @@ from sql.aggregate import Min from sql.functions import CurrentTimestamp, Extract from trytond.config import config -from trytond.model import ModelSQL, fields +from trytond.model import Index, ModelSQL, fields from trytond.pool import Pool from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -34,18 +34,17 @@ class Queue(ModelSQL): help="When the task should be done.") @classmethod - def __register__(cls, module_name): - queue = cls.__table__() - super().__register__(module_name) - table_h = cls.__table_handler__(module_name) + def __setup__(cls): + super().__setup__() + table = cls.__table__() - # Add index for candidates - table_h.index_action([ - queue.scheduled_at.nulls_first, - queue.expected_at.nulls_first, - queue.dequeued_at, - queue.name, - ], action='add') + cls._sql_indexes.add( + Index( + table, + (table.scheduled_at, Index.Range(nulls_first=True)), + (table.expected_at, Index.Range(nulls_first=True)), + (table.dequeued_at, Index.Equality()), + (table.name, Index.Equality()))) @classmethod def default_enqueued_at(cls): diff --git a/trytond/ir/resource.py b/trytond/ir/resource.py index d9138d4a9c..84b462f0b0 100644 --- a/trytond/ir/resource.py +++ b/trytond/ir/resource.py @@ -3,7 +3,7 @@ from sql.conditionals import Coalesce from trytond.i18n import lazy_gettext -from trytond.model import ModelStorage, ModelView, fields +from trytond.model import Index, ModelSQL, ModelStorage, ModelView, fields from trytond.pool import Pool from trytond.pyson import Eval from trytond.transaction import Transaction @@ -14,7 +14,15 @@ __all__ = ['ResourceAccessMixin', 'ResourceMixin', 'resource_copy'] class ResourceAccessMixin(ModelStorage): resource = fields.Reference( - "Resource", selection='get_models', required=True, select=True) + "Resource", selection='get_models', required=True) + + @classmethod + def __setup__(cls): + super().__setup__() + if issubclass(cls, ModelSQL): + table = cls.__table__() + cls._sql_indexes.add( + Index(table, (table.resource, Index.Similarity(begin=True)))) @classmethod def default_resource(cls): diff --git a/trytond/ir/rule.py b/trytond/ir/rule.py index 2eb710317b..9f42c96a74 100644 --- a/trytond/ir/rule.py +++ b/trytond/ir/rule.py @@ -6,7 +6,8 @@ from sql import Literal from trytond.cache import Cache from trytond.i18n import gettext -from trytond.model import Check, EvalEnvironment, ModelSQL, ModelView, fields +from trytond.model import ( + Check, EvalEnvironment, Index, ModelSQL, ModelView, fields) from trytond.model.exceptions import ValidationError from trytond.pool import Pool from trytond.pyson import PYSONDecoder @@ -21,18 +22,16 @@ class RuleGroup(ModelSQL, ModelView): "Rule group" __name__ = 'ir.rule.group' name = fields.Char( - "Name", select=True, translate=True, required=True, + "Name", translate=True, required=True, help="Displayed to users when access error is raised for this rule.") - model = fields.Many2One('ir.model', 'Model', select=True, + model = fields.Many2One('ir.model', 'Model', required=True, ondelete='CASCADE') - global_p = fields.Boolean('Global', select=True, + global_p = fields.Boolean('Global', help="Make the rule global \nso every users must follow this rule.") - default_p = fields.Boolean('Default', select=True, + default_p = fields.Boolean('Default', help="Add this rule to all users by default.") rules = fields.One2Many('ir.rule', 'rule_group', 'Tests', help="The rule is satisfied if at least one test is True.") - groups = fields.Many2Many('ir.rule.group-res.group', - 'rule_group', 'group', 'Groups') perm_read = fields.Boolean('Read Access') perm_write = fields.Boolean('Write Access') perm_create = fields.Boolean('Create Access') @@ -41,17 +40,21 @@ class RuleGroup(ModelSQL, ModelView): @classmethod def __setup__(cls): super(RuleGroup, cls).__setup__() + t = cls.__table__() + cls._order.insert(0, ('model', 'ASC')) cls._order.insert(1, ('global_p', 'ASC')) cls._order.insert(2, ('default_p', 'ASC')) - t = cls.__table__() cls._sql_constraints += [ ('global_default_exclusive', Check(t, (t.global_p == Literal(False)) | (t.default_p == Literal(False))), 'Global and Default are mutually exclusive!'), ] + cls._sql_indexes.update({ + Index(t, (t.model, Index.Equality())), + }) @staticmethod def default_global_p(): @@ -100,7 +103,7 @@ class RuleGroup(ModelSQL, ModelView): class Rule(ModelSQL, ModelView): "Rule" __name__ = 'ir.rule' - rule_group = fields.Many2One('ir.rule.group', 'Group', select=True, + rule_group = fields.Many2One('ir.rule.group', 'Group', required=True, ondelete="CASCADE") domain = fields.Char('Domain', required=True, help="Domain is evaluated with a PYSON context containing:" @@ -110,6 +113,14 @@ class Rule(ModelSQL, ModelView): modes = {'read', 'write', 'create', 'delete'} + @classmethod + def __setup__(cls): + super().__setup__() + table = cls.__table__() + + cls._sql_indexes.add( + Index(table, (table.rule_group, Index.Equality()))) + @classmethod def validate_fields(cls, rules, field_names): super().validate_fields(rules, field_names) diff --git a/trytond/ir/session.py b/trytond/ir/session.py index 8ffd473ded..8c01334f60 100644 --- a/trytond/ir/session.py +++ b/trytond/ir/session.py @@ -6,7 +6,7 @@ from secrets import token_hex from trytond.cache import Cache from trytond.config import config -from trytond.model import ModelSQL, fields +from trytond.model import Index, ModelSQL, fields _session_timeout = datetime.timedelta( seconds=config.getint('session', 'timeout')) @@ -18,20 +18,26 @@ class Session(ModelSQL): __name__ = 'ir.session' _rec_name = 'key' - key = fields.Char('Key', required=True, select=True, strip=False) + key = fields.Char("Key", required=True, strip=False) _session_reset_cache = Cache('ir_session.session_reset', context=False) @classmethod def __setup__(cls): super(Session, cls).__setup__() + table = cls.__table__() cls.__rpc__ = {} - - @classmethod - def __register__(cls, module_name): - super(Session, cls).__register__(module_name) - - table = cls.__table_handler__(module_name) - table.index_action('create_uid', 'add') + cls.__rpc__ = {} + cls._sql_indexes.update({ + Index(table, + (table.key, Index.Equality()), + (table.create_uid, Index.Equality())), + Index(table, + (table.key, Index.Equality()), + (table.create_date, Index.Equality())), + Index(table, + (table.key, Index.Equality()), + (table.write_date, Index.Equality())), + }) @classmethod def default_key(cls, nbytes=None): diff --git a/trytond/ir/translation.py b/trytond/ir/translation.py index 9436ead19d..c2d8992823 100644 --- a/trytond/ir/translation.py +++ b/trytond/ir/translation.py @@ -20,7 +20,7 @@ from trytond.cache import Cache from trytond.config import config from trytond.exceptions import UserError from trytond.i18n import gettext -from trytond.model import ModelSQL, ModelView, fields +from trytond.model import Index, ModelSQL, ModelView, fields from trytond.pool import Pool from trytond.pyson import Eval, PYSONEncoder from trytond.tools import cursor_dict, file_open, grouped_slice @@ -58,7 +58,7 @@ class Translation(ModelSQL, ModelView): __name__ = "ir.translation" name = fields.Char('Field Name', required=True) - res_id = fields.Integer('Resource ID', select=True, required=True) + res_id = fields.Integer('Resource ID', required=True) lang = fields.Selection('get_language', string='Language') type = fields.Selection(TRANSLATION_TYPE, string='Type', required=True) @@ -75,6 +75,37 @@ class Translation(ModelSQL, ModelView): 'ir.translation.get_report', context=False) _get_language_cache = Cache('ir.translation.get_language', context=False) + @classmethod + def __setup__(cls): + cls.name.search_unaccented = False + super().__setup__() + table = cls.__table__() + + cls._sql_indexes.update({ + Index( + table, (Index.Unaccent(table.src), Index.Similarity())), + Index( + table, (Index.Unaccent(table.value), Index.Similarity())), + Index( + table, + (table.type, Index.Equality()), + (table.name, Index.Equality()), + (table.lang, Index.Equality()), + (table.res_id, Index.Range()), + (table.fuzzy, Index.Equality()), + (Index.Unaccent(table.value), Index.Similarity())), + Index( + table, + (table.type, Index.Equality()), + (table.name, Index.Equality())), + Index( + table, + (table.res_id, Index.Equality()), + (table.name, Index.Equality()), + (table.lang, Index.Equality()), + (table.type, Index.Equality())), + }) + @classmethod def __register__(cls, module_name): transaction = Transaction() @@ -95,9 +126,6 @@ class Translation(ModelSQL, ModelView): ['report'], where=ir_translation.type == 'odt')) - table = cls.__table_handler__(module_name) - table.index_action(['lang', 'type', 'name'], 'add') - @classmethod def register_model(cls, model, module_name): cursor = Transaction().connection.cursor() diff --git a/trytond/ir/trigger.py b/trytond/ir/trigger.py index c02f5e1598..b12287482b 100644 --- a/trytond/ir/trigger.py +++ b/trytond/ir/trigger.py @@ -11,7 +11,8 @@ from sql.operators import Concat from trytond.cache import Cache from trytond.i18n import gettext from trytond.model import ( - Check, DeactivableMixin, EvalEnvironment, ModelSQL, ModelView, fields) + Check, DeactivableMixin, EvalEnvironment, Index, ModelSQL, ModelView, + fields) from trytond.model.exceptions import ValidationError from trytond.pool import Pool from trytond.pyson import Eval, PYSONDecoder @@ -27,19 +28,19 @@ class Trigger(DeactivableMixin, ModelSQL, ModelView): "Trigger" __name__ = 'ir.trigger' name = fields.Char('Name', required=True, translate=True) - model = fields.Many2One('ir.model', 'Model', required=True, select=True) - on_time = fields.Boolean('On Time', select=True, states={ + model = fields.Many2One('ir.model', 'Model', required=True) + on_time = fields.Boolean('On Time', states={ 'invisible': (Eval('on_create', False) | Eval('on_write', False) | Eval('on_delete', False)), }, depends=['on_create', 'on_write', 'on_delete']) - on_create = fields.Boolean('On Create', select=True, states={ + on_create = fields.Boolean('On Create', states={ 'invisible': Eval('on_time', False), }, depends=['on_time']) - on_write = fields.Boolean('On Write', select=True, states={ + on_write = fields.Boolean('On Write', states={ 'invisible': Eval('on_time', False), }, depends=['on_time']) - on_delete = fields.Boolean('On Delete', select=True, states={ + on_delete = fields.Boolean('On Delete', states={ 'invisible': Eval('on_time', False), }, depends=['on_time']) condition = fields.Char('Condition', required=True, @@ -314,8 +315,12 @@ class TriggerLog(ModelSQL): record_id = fields.Integer('Record ID', required=True) @classmethod - def __register__(cls, module_name): - super(TriggerLog, cls).__register__(module_name) + def __setup__(cls): + super().__setup__() - table = cls.__table_handler__(module_name) - table.index_action(['trigger', 'record_id'], 'add') + table = cls.__table__() + cls._sql_indexes.add( + Index( + table, + (table.trigger, Index.Equality()), + (table.record_id, Index.Range()))) diff --git a/trytond/ir/ui/icon.py b/trytond/ir/ui/icon.py index 9660cd8067..3063934450 100644 --- a/trytond/ir/ui/icon.py +++ b/trytond/ir/ui/icon.py @@ -14,7 +14,7 @@ class Icon(sequence_ordered(), ModelSQL, ModelView): 'Icon' __name__ = 'ir.ui.icon' - name = fields.Char('Name', required=True, select=True) + name = fields.Char('Name', required=True) module = fields.Char('Module', readonly=True, required=True) path = fields.Char('SVG Path', readonly=True, required=True) icon = fields.Function(fields.Char('Icon', depends=['path']), 'get_icon') diff --git a/trytond/ir/ui/menu.py b/trytond/ir/ui/menu.py index 191f8a2b56..02014a3923 100644 --- a/trytond/ir/ui/menu.py +++ b/trytond/ir/ui/menu.py @@ -85,10 +85,7 @@ class UIMenu( name = fields.Char('Menu', required=True, translate=True) childs = fields.One2Many('ir.ui.menu', 'parent', 'Children') - parent = fields.Many2One('ir.ui.menu', 'Parent Menu', select=True, - ondelete='CASCADE') - groups = fields.Many2Many('ir.ui.menu-res.group', - 'menu', 'group', 'Groups') + parent = fields.Many2One('ir.ui.menu', 'Parent Menu', ondelete='CASCADE') complete_name = fields.Function(fields.Char('Complete Name'), 'get_rec_name', searcher='search_rec_name') icon = fields.Selection('list_icons', 'Icon', translate=False) diff --git a/trytond/ir/ui/view.py b/trytond/ir/ui/view.py index 9cb0e1c04d..7e54225d42 100644 --- a/trytond/ir/ui/view.py +++ b/trytond/ir/ui/view.py @@ -7,7 +7,7 @@ from lxml import etree from trytond.cache import Cache from trytond.i18n import gettext -from trytond.model import ModelSQL, ModelView, fields +from trytond.model import Index, ModelSQL, ModelView, fields from trytond.model.exceptions import ValidationError from trytond.pool import Pool from trytond.pyson import PYSON, Bool, Eval, If, PYSONDecoder @@ -27,10 +27,10 @@ class View(ModelSQL, ModelView): "View" __name__ = 'ir.ui.view' _rec_name = 'model' - model = fields.Char('Model', select=True, states={ + model = fields.Char('Model', states={ 'required': Eval('type') != 'board', }) - priority = fields.Integer('Priority', required=True, select=True) + priority = fields.Integer('Priority', required=True) type = fields.Selection([ (None, ''), ('tree', 'Tree'), @@ -39,7 +39,7 @@ class View(ModelSQL, ModelView): ('calendar', 'Calendar'), ('board', 'Board'), ('list-form', "List Form"), - ], 'View Type', select=True, + ], 'View Type', domain=[ If(Bool(Eval('inherit')), ('type', '=', None), @@ -54,7 +54,7 @@ class View(ModelSQL, ModelView): arch = fields.Function(fields.Text('View Architecture', states={ 'readonly': Bool(Eval('name')), }, depends=['name']), 'get_arch', setter='set_arch') - inherit = fields.Many2One('ir.ui.view', 'Inherited View', select=True, + inherit = fields.Many2One('ir.ui.view', 'Inherited View', ondelete='CASCADE') field_childs = fields.Char('Children Field', states={ 'invisible': Eval('type') != 'tree', @@ -72,6 +72,8 @@ class View(ModelSQL, ModelView): @classmethod def __setup__(cls): super(View, cls).__setup__() + table = cls.__table__() + cls.__rpc__['view_get'] = RPC(instantiate=0, cache=dict(days=1)) cls._order.insert(0, ('priority', 'ASC')) cls._buttons.update({ @@ -80,6 +82,15 @@ class View(ModelSQL, ModelView): 'depends': ['type'], }, }) + cls._sql_indexes.update({ + Index(table, + (table.model, Index.Equality()), + (table.inherit, Index.Equality())), + Index( + table, + (table.id, Index.Equality()), + (table.inherit, Index.Equality())), + }) @staticmethod def default_priority(): @@ -368,18 +379,25 @@ class ViewTreeWidth(ModelSQL, ModelView): "View Tree Width" __name__ = 'ir.ui.view_tree_width' _rec_name = 'model' - model = fields.Char('Model', required=True, select=True) - field = fields.Char('Field', required=True, select=True) + model = fields.Char('Model', required=True) + field = fields.Char('Field', required=True) user = fields.Many2One('res.user', 'User', required=True, - ondelete='CASCADE', select=True) + ondelete='CASCADE') width = fields.Integer('Width', required=True) @classmethod def __setup__(cls): super(ViewTreeWidth, cls).__setup__() + table = cls.__table__() cls.__rpc__.update({ 'set_width': RPC(readonly=False), }) + cls._sql_indexes.add( + Index( + table, + (table.user, Index.Equality()), + (table.model, Index.Equality()), + (table.field, Index.Equality()))) @classmethod def delete(cls, records): @@ -441,12 +459,12 @@ class ViewTreeOptional(ModelSQL, ModelView): cls.__rpc__.update({ 'set_optional': RPC(readonly=False), }) - - @classmethod - def __register__(cls, module_name): - super().__register__(module_name) - table_h = cls.__table_handler__(module_name) - table_h.index_action(['view_id', 'user'], 'add') + table = cls.__table__() + cls._sql_indexes.add( + Index( + table, + (table.user, Index.Equality()), + (table.view_id, Index.Equality()))) @classmethod def create(cls, vlist): @@ -505,12 +523,14 @@ class ViewTreeState(ModelSQL, ModelView): 'get': RPC(check_access=False, cache=dict(days=1)), }) - @classmethod - def __register__(cls, module_name): - super(ViewTreeState, cls).__register__(module_name) - - table = cls.__table_handler__(module_name) - table.index_action(['model', 'domain', 'user', 'child_name'], 'add') + table = cls.__table__() + cls._sql_indexes.add( + Index( + table, + (table.user, Index.Equality()), + (table.model, Index.Equality()), + (table.child_name, Index.Equality()), + (table.domain, Index.Equality()))) @staticmethod def default_nodes(): diff --git a/trytond/model/__init__.py b/trytond/model/__init__.py index c535b00e39..1df6f35a78 100644 --- a/trytond/model/__init__.py +++ b/trytond/model/__init__.py @@ -8,7 +8,7 @@ from .digits import DigitsMixin from .match import MatchMixin from .model import Model from .modelsingleton import ModelSingleton -from .modelsql import Check, Exclude, ModelSQL, Unique +from .modelsql import Check, Exclude, Index, ModelSQL, Unique from .modelstorage import EvalEnvironment, ModelStorage from .modelview import ModelView from .multivalue import MultiValueMixin, ValueMixin @@ -19,7 +19,7 @@ from .union import UnionMixin from .workflow import Workflow __all__ = ['Model', 'ModelView', 'ModelStorage', 'ModelSingleton', 'ModelSQL', - 'Check', 'Unique', 'Exclude', + 'Check', 'Unique', 'Exclude', 'Index', 'Workflow', 'DictSchemaMixin', 'MatchMixin', 'UnionMixin', 'dualmethod', 'MultiValueMixin', 'ValueMixin', 'SymbolMixin', 'DigitsMixin', 'EvalEnvironment', 'sequence_ordered', 'DeactivableMixin', 'tree', diff --git a/trytond/model/fields/binary.py b/trytond/model/fields/binary.py index 5dabc25b57..feaf1bf2fe 100644 --- a/trytond/model/fields/binary.py +++ b/trytond/model/fields/binary.py @@ -26,7 +26,7 @@ class Binary(Field): cast = staticmethod(caster) def __init__(self, string='', help='', required=False, readonly=False, - domain=None, states=None, select=False, on_change=None, + domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='lazy', filename=None, file_id=None, store_prefix=None): self.filename = filename @@ -34,7 +34,7 @@ class Binary(Field): self.store_prefix = store_prefix super(Binary, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) @cached_property diff --git a/trytond/model/fields/boolean.py b/trytond/model/fields/boolean.py index 070f0e4980..b05fed54b6 100644 --- a/trytond/model/fields/boolean.py +++ b/trytond/model/fields/boolean.py @@ -13,10 +13,10 @@ class Boolean(Field): _py_type = bool def __init__(self, string='', help='', readonly=False, domain=None, - states=None, select=False, on_change=None, on_change_with=None, + states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): super(Boolean, self).__init__(string=string, help=help, required=False, - readonly=readonly, domain=domain, states=states, select=select, + readonly=readonly, domain=domain, states=states, on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) diff --git a/trytond/model/fields/char.py b/trytond/model/fields/char.py index 09e8bf7d9e..1d113d9fcf 100644 --- a/trytond/model/fields/char.py +++ b/trytond/model/fields/char.py @@ -27,8 +27,8 @@ class Char(FieldTranslate): def __init__(self, string='', size=None, help='', required=False, readonly=False, domain=None, states=None, translate=False, - select=False, on_change=None, on_change_with=None, depends=None, - context=None, loading=None, autocomplete=None, strip=True): + on_change=None, on_change_with=None, depends=None, context=None, + loading=None, autocomplete=None, strip=True): ''' :param translate: A boolean. If ``True`` the field is translatable. :param size: A integer. If set defines the maximum size of the values. @@ -36,7 +36,7 @@ class Char(FieldTranslate): if loading is None: loading = 'lazy' if translate else 'eager' super(Char, self).__init__(string=string, help=help, required=required, - readonly=readonly, domain=domain, states=states, select=select, + readonly=readonly, domain=domain, states=states, on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.autocomplete = set() diff --git a/trytond/model/fields/date.py b/trytond/model/fields/date.py index d8cc9a4bdd..966eecb531 100644 --- a/trytond/model/fields/date.py +++ b/trytond/model/fields/date.py @@ -102,7 +102,7 @@ class DateTime(Timestamp): _sql_type = 'DATETIME' def __init__(self, string='', format='%H:%M:%S', help='', required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' @@ -110,7 +110,7 @@ class DateTime(Timestamp): ''' super(DateTime, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.format = format @@ -132,7 +132,7 @@ class Time(FormatMixin, Field): _py_type = datetime.time def __init__(self, string='', format='%H:%M:%S', help='', required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' @@ -140,7 +140,7 @@ class Time(FormatMixin, Field): ''' super().__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.format = format @@ -167,7 +167,7 @@ class TimeDelta(Field): _py_type = datetime.timedelta def __init__(self, string='', converter=None, help='', required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' @@ -176,7 +176,7 @@ class TimeDelta(Field): ''' super(TimeDelta, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.converter = converter diff --git a/trytond/model/fields/dict.py b/trytond/model/fields/dict.py index 0f4371d8c1..6b8d856098 100644 --- a/trytond/model/fields/dict.py +++ b/trytond/model/fields/dict.py @@ -27,12 +27,11 @@ class Dict(Field): _py_type = dict def __init__(self, schema_model, string='', help='', required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='lazy'): super(Dict, self).__init__(string, help, required, readonly, domain, - states, select, on_change, on_change_with, depends, context, - loading) + states, on_change, on_change_with, depends, context, loading) self.schema_model = schema_model self.search_unaccented = True diff --git a/trytond/model/fields/field.py b/trytond/model/fields/field.py index 81ba21c620..2efb82adc3 100644 --- a/trytond/model/fields/field.py +++ b/trytond/model/fields/field.py @@ -231,7 +231,7 @@ class Field(object): _py_type = None def __init__(self, string='', help='', required=False, readonly=False, - domain=None, states=None, select=False, on_change=None, + domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' @@ -245,7 +245,6 @@ class Field(object): ``readonly`` and ``invisible``. Values are pyson expressions that will be evaluated with record values. This allows to change dynamically the attributes of the field. - :param select: An boolean. When True search will be optimized. :param on_change: A list of values. If set, the client will call the method ``on_change_`` when the user changes the field value. It then passes this list of values as arguments to the @@ -269,7 +268,6 @@ class Field(object): self.domain = domain self.__states = None self.states = states or {} - self.select = bool(select) self.on_change = set() if on_change: warnings.warn('on_change argument is deprecated, ' diff --git a/trytond/model/fields/float.py b/trytond/model/fields/float.py index b1b5a6396f..b14e0d1733 100644 --- a/trytond/model/fields/float.py +++ b/trytond/model/fields/float.py @@ -35,7 +35,7 @@ class Float(Field): _py_type = float def __init__(self, string='', digits=None, help='', required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' @@ -44,7 +44,7 @@ class Float(Field): ''' super(Float, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.__digits = None self.digits = digits diff --git a/trytond/model/fields/many2one.py b/trytond/model/fields/many2one.py index 57055bb942..cbddacb5bb 100644 --- a/trytond/model/fields/many2one.py +++ b/trytond/model/fields/many2one.py @@ -25,7 +25,7 @@ class Many2One(Field): def __init__(self, model_name, string='', left=None, right=None, path=None, ondelete='SET NULL', datetime_field=None, target_search='join', search_order=None, search_context=None, help='', required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' @@ -50,7 +50,7 @@ class Many2One(Field): self.ondelete = ondelete super(Many2One, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.model_name = model_name self.left = left diff --git a/trytond/model/fields/multiselection.py b/trytond/model/fields/multiselection.py index 160db02e4b..1142e31340 100644 --- a/trytond/model/fields/multiselection.py +++ b/trytond/model/fields/multiselection.py @@ -23,7 +23,7 @@ class MultiSelection(SelectionMixin, Field): def __init__(self, selection, string='', sort=True, translate=True, help='', help_selection=None, required=False, readonly=False, - domain=None, states=None, select=False, on_change=None, + domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): """ :param selection: A list or a function name that returns a list. @@ -33,7 +33,7 @@ class MultiSelection(SelectionMixin, Field): """ super().__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) if hasattr(selection, 'copy'): self.selection = selection.copy() diff --git a/trytond/model/fields/reference.py b/trytond/model/fields/reference.py index 66ab69f934..33b44569fb 100644 --- a/trytond/model/fields/reference.py +++ b/trytond/model/fields/reference.py @@ -27,7 +27,7 @@ class Reference(SelectionMixin, Field): def __init__(self, string='', selection=None, sort=True, selection_change_with=None, translate=True, search_order=None, search_context=None, help='', help_selection=None, required=False, - readonly=False, domain=None, states=None, select=False, + readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='lazy', datetime_field=None): ''' @@ -42,7 +42,7 @@ class Reference(SelectionMixin, Field): ''' super(Reference, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) self.datetime_field = datetime_field self.selection = selection or None diff --git a/trytond/model/fields/selection.py b/trytond/model/fields/selection.py index 4711204575..498076e2b4 100644 --- a/trytond/model/fields/selection.py +++ b/trytond/model/fields/selection.py @@ -83,7 +83,7 @@ class Selection(SelectionMixin, Field): def __init__(self, selection, string='', sort=True, selection_change_with=None, translate=True, help='', help_selection=None, required=False, readonly=False, domain=None, - states=None, select=False, on_change=None, on_change_with=None, + states=None, on_change=None, on_change_with=None, depends=None, context=None, loading='eager'): ''' :param selection: A list or a function name that returns a list. @@ -93,7 +93,7 @@ class Selection(SelectionMixin, Field): ''' super(Selection, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, - select=select, on_change=on_change, on_change_with=on_change_with, + on_change=on_change, on_change_with=on_change_with, depends=depends, context=context, loading=loading) if hasattr(selection, 'copy'): self.selection = selection.copy() diff --git a/trytond/model/modelsql.py b/trytond/model/modelsql.py index 6b24fbf520..66bc79f18e 100644 --- a/trytond/model/modelsql.py +++ b/trytond/model/modelsql.py @@ -153,6 +153,78 @@ class Exclude(Constraint): return tuple(p) +class Index: + __slots__ = ('table', 'expressions', 'options') + + def __init__(self, table, *expressions, **options): + self.table = table + assert all( + isinstance(e, Expression) and isinstance(u, self.Usage) + for e, u in expressions) + self.expressions = expressions + self.options = options + + def __hash__(self): + table_def = ( + self.table._name, self.table._schema, self.table._database) + expressions = ( + (str(e), e.params, hash(u)) for e, u in self.expressions) + return hash((table_def, *expressions)) + + def __eq__(self, other): + return ( + str(self.table) == str(other.table) + and len(self.expressions) == len(other.expressions) + and all((str(c), u) == (str(oc), ou) + for (c, u), (oc, ou) in zip( + self.expressions, other.expressions)) + and self.options == other.options) + + class Unaccent(Expression): + "Unaccent function if database support for index" + __slots__ = ('_expression',) + + def __init__(self, expression): + self._expression = expression + + @property + def expression(self): + expression = self._expression + database = Transaction().database + if database.has_unaccent_indexable(): + expression = database.unaccent(expression) + return expression + + def __str__(self): + return str(self.expression) + + @property + def params(self): + return self.expression.params + + class Usage: + __slots__ = ('options',) + + def __init__(self, **options): + self.options = options + + def __hash__(self): + return hash((self.__class__.__name__, *self.options.items())) + + def __eq__(self, other): + return (self.__class__ == other.__class__ + and self.options == other.options) + + class Equality(Usage): + __slots__ = () + + class Range(Usage): + __slots__ = () + + class Similarity(Usage): + __slots__ = () + + def no_table_query(func): @wraps(func) def wrapper(cls, *args, **kwargs): @@ -185,16 +257,72 @@ class ModelSQL(ModelStorage): super(ModelSQL, cls).__setup__() cls._sql_constraints = [] + cls._sql_indexes = set() if not callable(cls.table_query): table = cls.__table__() cls._sql_constraints.append( ('id_positive', Check(table, table.id >= 0), 'ir.msg_id_positive')) + rec_name_field = getattr(cls, cls._rec_name, None) + if (isinstance(rec_name_field, fields.Field) + and not hasattr(rec_name_field, 'set')): + column = Column(table, cls._rec_name) + if getattr(rec_name_field, 'search_unaccented', False): + column = Index.Unaccent(column) + cls._sql_indexes.add( + Index(table, (column, Index.Similarity()))) cls._order = [('id', None)] if issubclass(cls, ModelView): cls.__rpc__.update({ 'history_revisions': RPC(), }) + if cls._history: + history_table = cls.__table_history__() + cls._sql_indexes.add( + Index( + history_table, + (history_table.id, Index.Equality()))) + + @classmethod + def __post_setup__(cls): + super().__post_setup__() + + # Define Range index to optimise with reduce_ids + for field in cls._fields.values(): + if isinstance(field, fields.One2Many) and field.field: + Target = field.get_target() + field_name = field.field + elif isinstance(field, fields.Many2Many) and field.origin: + Target = field.get_relation() + field_name = field.origin + else: + continue + if field_name == 'id': + continue + target_field = getattr(Target, field_name) + if (issubclass(Target, ModelSQL) + and not callable(Target.table_query) + and not hasattr(target_field, 'set')): + target = Target.__table__() + column = Column(target, field_name) + if not target_field.required and Target != cls: + where = column != Null + else: + where = None + if target_field._type == 'reference': + Target._sql_indexes.update({ + Index( + target, + (column, Index.Equality()), + where=where), + Index( + target, + (column, Index.Similarity(begin=True)), + where=where), + }) + else: + Target._sql_indexes.add( + Index(target, (column, Index.Range()), where=where)) @classmethod def __table__(cls): @@ -230,7 +358,6 @@ class ModelSQL(ModelStorage): table = cls.__table_handler__(module_name) if cls._history: history_table = cls.__table_handler__(module_name, history=True) - history_table.index_action('id', action='add') for field_name, field in cls._fields.items(): if field_name == 'id': @@ -278,9 +405,6 @@ class ModelSQL(ModelStorage): elif ref: table.add_fk(field_name, ref, field.ondelete) - table.index_action( - field_name, action=field.select and 'add' or 'remove') - required = field.required # Do not set 'NOT NULL' for Binary field as the database column # will be left empty if stored in the filestore or filled later by @@ -319,6 +443,8 @@ class ModelSQL(ModelStorage): cls._rebuild_tree(field_name, None, 0) for ident, constraint, _ in cls._sql_constraints: + assert ( + not ident.startswith('idx_') and not ident.endswith('_index')) table.add_constraint(ident, constraint) if cls._history: @@ -338,6 +464,13 @@ class ModelSQL(ModelStorage): cursor.execute(*history_table.update( [history_table.write_date], [None])) + @classmethod + def _update_sql_indexes(cls): + if not callable(cls.table_query): + table_h = cls.__table_handler__() + # TODO: remove overlapping indexes + table_h.set_indexes(cls._sql_indexes) + @classmethod def _update_history_table(cls): if cls._history: diff --git a/trytond/model/order.py b/trytond/model/order.py index 0ea6024739..73bdb010e9 100644 --- a/trytond/model/order.py +++ b/trytond/model/order.py @@ -1,7 +1,9 @@ # This file is part of Tryton. The COPYRIGHT file at the toplevel of this # repository contains the full copyright notices and license terms. +from sql import Column + from trytond.i18n import lazy_gettext -from trytond.model import fields +from trytond.model import Index, fields def sequence_ordered( @@ -18,7 +20,12 @@ def sequence_ordered( @classmethod def __setup__(cls): super(SequenceOrderedMixin, cls).__setup__() + table = cls.__table__() cls._order = [(field_name, order)] + cls._order + cls._sql_indexes.add( + Index(table, + (Column(table, field_name), Index.Range(order=order)), + (table.id, Index.Range(order=order)))) setattr(SequenceOrderedMixin, field_name, fields.Integer(field_label)) return SequenceOrderedMixin diff --git a/trytond/modules/__init__.py b/trytond/modules/__init__.py index e29140d8ff..7f56a7bf09 100644 --- a/trytond/modules/__init__.py +++ b/trytond/modules/__init__.py @@ -195,6 +195,7 @@ def load_module_graph(graph, pool, update=None, lang=None): update = [] modules_todo = [] models_to_update_history = set() + models_with_indexes = set() # Load also parent languages lang = set(lang) @@ -239,6 +240,8 @@ def load_module_graph(graph, pool, update=None, lang=None): for model in classes['model']: if hasattr(model, '_history'): models_to_update_history.add(model.__name__) + if hasattr(model, '_update_sql_indexes'): + models_with_indexes.add(model.__name__) # Instanciate a new parser for the module tryton_parser = convert.TrytondXmlHandler( @@ -289,11 +292,17 @@ def load_module_graph(graph, pool, update=None, lang=None): pool.setup_mixin() - for model_name in models_to_update_history: - model = pool.get(model_name) - if model._history: - logger.info('history:update %s', model.__name__) - model._update_history_table() + if update: + for model_name in models_with_indexes: + model = pool.get(model_name) + if model._sql_indexes: + logger.info('index:create %s', model_name) + model._update_sql_indexes() + for model_name in models_to_update_history: + model = pool.get(model_name) + if model._history: + logger.info('history:update %s', model.__name__) + model._update_history_table() # Vacuum : while modules_todo: diff --git a/trytond/res/__init__.py b/trytond/res/__init__.py index 84d3332e9d..b3bc6a8a7c 100644 --- a/trytond/res/__init__.py +++ b/trytond/res/__init__.py @@ -18,6 +18,7 @@ def register(): user.Warning_, user.UserApplication, user.UserConfigStart, + ir.UIMenu, ir.UIMenuGroup, ir.ActionGroup, ir.Action, @@ -26,9 +27,11 @@ def register(): ir.ActionWizard, ir.ActionURL, ir.ActionKeyword, + ir.ModelButton, ir.ModelButtonGroup, ir.ModelButtonRule, ir.ModelButtonClick, + ir.RuleGroup, ir.RuleGroupGroup, ir.SequenceType, ir.SequenceTypeGroup, diff --git a/trytond/res/group.py b/trytond/res/group.py index 43f3a12feb..d393a014b2 100644 --- a/trytond/res/group.py +++ b/trytond/res/group.py @@ -31,7 +31,7 @@ class MenuMany2Many(fields.Many2Many): class Group(DeactivableMixin, tree(), ModelSQL, ModelView): "Group" __name__ = "res.group" - name = fields.Char('Name', required=True, select=True, translate=True) + name = fields.Char('Name', required=True, translate=True) users = fields.Many2Many('res.user-res.group', 'group', 'user', 'Users') parent = fields.Many2One( 'res.group', "Parent", diff --git a/trytond/res/ir.py b/trytond/res/ir.py index 976f6d1309..2b9965ae92 100644 --- a/trytond/res/ir.py +++ b/trytond/res/ir.py @@ -1,17 +1,26 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. + +from trytond.cache import Cache from trytond.model import DeactivableMixin, ModelSQL, fields from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval +class UIMenu(metaclass=PoolMeta): + __name__ = 'ir.ui.menu' + + groups = fields.Many2Many( + 'ir.ui.menu-res.group', 'menu', 'group', "Groups") + + class UIMenuGroup(ModelSQL): "UI Menu - Group" __name__ = 'ir.ui.menu-res.group' menu = fields.Many2One('ir.ui.menu', 'Menu', ondelete='CASCADE', - select=True, required=True) + required=True) group = fields.Many2One('res.group', 'Group', ondelete='CASCADE', - select=True, required=True) + required=True) @classmethod def create(cls, vlist): @@ -37,9 +46,9 @@ class ActionGroup(ModelSQL): "Action - Group" __name__ = 'ir.action-res.group' action = fields.Many2One('ir.action', 'Action', ondelete='CASCADE', - select=True, required=True) + required=True) group = fields.Many2One('res.group', 'Group', ondelete='CASCADE', - select=True, required=True) + required=True) @classmethod def create(cls, vlist): @@ -126,13 +135,58 @@ class ActionKeyword(metaclass=PoolMeta): return [('action.' + clause[0],) + tuple(clause[1:])] +class ModelButton(metaclass=PoolMeta): + __name__ = 'ir.model.button' + + groups = fields.Many2Many( + 'ir.model.button-res.group', 'button', 'group', "Groups") + _groups_cache = Cache('ir.model.button.groups') + + @classmethod + def create(cls, vlist): + result = super().create(vlist) + cls._groups_cache.clear() + return result + + @classmethod + def write(cls, buttons, values, *args): + super().write(buttons, values, *args) + cls._groups_cache.clear() + + @classmethod + def delete(cls, buttons): + super().delete(buttons) + cls._groups_cache.clear() + + @classmethod + def get_groups(cls, model, name): + ''' + Return a set of group ids for the named button on the model. + ''' + key = (model, name) + groups = cls._groups_cache.get(key) + if groups is not None: + return groups + buttons = cls.search([ + ('model.model', '=', model), + ('name', '=', name), + ]) + if not buttons: + groups = set() + else: + button, = buttons + groups = set(g.id for g in button.groups) + cls._groups_cache.set(key, groups) + return groups + + class ModelButtonGroup(DeactivableMixin, ModelSQL): "Model Button - Group" __name__ = 'ir.model.button-res.group' button = fields.Many2One('ir.model.button', 'Button', - ondelete='CASCADE', select=True, required=True) + ondelete='CASCADE', required=True) group = fields.Many2One('res.group', 'Group', ondelete='CASCADE', - select=True, required=True) + required=True) @classmethod def create(cls, vlist): @@ -167,13 +221,20 @@ class ModelButtonClick(metaclass=PoolMeta): user = fields.Many2One('res.user', "User", ondelete='CASCADE') +class RuleGroup(metaclass=PoolMeta): + __name__ = 'ir.rule.group' + + groups = fields.Many2Many( + 'ir.rule.group-res.group', 'rule_group', 'group', "Groups") + + class RuleGroupGroup(ModelSQL): "Rule Group - Group" __name__ = 'ir.rule.group-res.group' rule_group = fields.Many2One('ir.rule.group', 'Rule Group', - ondelete='CASCADE', select=True, required=True) + ondelete='CASCADE', required=True) group = fields.Many2One('res.group', 'Group', ondelete='CASCADE', - select=True, required=True) + required=True) class SequenceType(metaclass=PoolMeta): @@ -187,9 +248,9 @@ class SequenceTypeGroup(ModelSQL): 'Sequence Type - Group' __name__ = 'ir.sequence.type-res.group' sequence_type = fields.Many2One('ir.sequence.type', 'Sequence Type', - ondelete='CASCADE', select=True, required=True) + ondelete='CASCADE', required=True) group = fields.Many2One('res.group', 'User Groups', - ondelete='CASCADE', select=True, required=True) + ondelete='CASCADE', required=True) @classmethod def delete(cls, records): @@ -238,7 +299,7 @@ class Export_Group(ModelSQL): __name__ = 'ir.export-res.group' export = fields.Many2One( - 'ir.export', "Export", required=True, select=True, ondelete='CASCADE') + 'ir.export', "Export", required=True, ondelete='CASCADE') group = fields.Many2One( 'res.group', "Group", required=True, ondelete='CASCADE') diff --git a/trytond/res/user.py b/trytond/res/user.py index 20fe1bcfe2..e6ef660352 100644 --- a/trytond/res/user.py +++ b/trytond/res/user.py @@ -46,8 +46,8 @@ from trytond.config import config from trytond.exceptions import LoginException, RateLimitException, UserError from trytond.i18n import gettext from trytond.model import ( - DeactivableMixin, ModelSQL, ModelView, Unique, Workflow, avatar_mixin, - fields) + DeactivableMixin, Index, ModelSQL, ModelView, Unique, Workflow, + avatar_mixin, fields) from trytond.pool import Pool from trytond.pyson import Bool, Eval, PYSONEncoder from trytond.report import Report, get_email @@ -110,7 +110,7 @@ class DeleteError(UserError): class User(avatar_mixin(100, 'login'), DeactivableMixin, ModelSQL, ModelView): "User" __name__ = "res.user" - name = fields.Char('Name', select=True) + name = fields.Char('Name') login = fields.Char('Login', required=True) password_hash = fields.Char('Password Hash', strip=False) password = fields.Function(fields.Char( @@ -950,10 +950,10 @@ class UserDevice(ModelSQL): class UserAction(ModelSQL): 'User - Action' __name__ = 'res.user-ir.action' - user = fields.Many2One('res.user', 'User', ondelete='CASCADE', select=True, + user = fields.Many2One('res.user', 'User', ondelete='CASCADE', required=True) action = fields.Many2One('ir.action', 'Action', ondelete='CASCADE', - select=True, required=True) + required=True) @staticmethod def _convert_values(values): @@ -981,10 +981,10 @@ class UserAction(ModelSQL): class UserGroup(ModelSQL): 'User - Group' __name__ = 'res.user-res.group' - user = fields.Many2One('res.user', 'User', ondelete='CASCADE', select=True, - required=True) + user = fields.Many2One('res.user', 'User', ondelete='CASCADE', + required=True) group = fields.Many2One('res.group', 'Group', ondelete='CASCADE', - select=True, required=True) + required=True) @classmethod def create(cls, vlist): @@ -1047,8 +1047,8 @@ class Warning_(ModelSQL, ModelView): 'User Warning' __name__ = 'res.user.warning' - user = fields.Many2One('res.user', 'User', required=True, select=True) - name = fields.Char('Name', required=True, select=True) + user = fields.Many2One('res.user', 'User', required=True) + name = fields.Char('Name', required=True) always = fields.Boolean('Always') @classmethod @@ -1082,8 +1082,8 @@ class UserApplication(Workflow, ModelSQL, ModelView): __name__ = 'res.user.application' _rec_name = 'key' - key = fields.Char("Key", required=True, select=True, strip=False) - user = fields.Many2One('res.user', "User", select=True) + key = fields.Char("Key", required=True, strip=False) + user = fields.Many2One('res.user', "User") application = fields.Selection([], "Application") state = fields.Selection([ ('requested', "Requested"), @@ -1112,6 +1112,20 @@ class UserApplication(Workflow, ModelSQL, ModelView): # Do not cache default_key as it depends on time cls.__rpc__['default_get'].cache = None + table = cls.__table__() + cls._sql_indexes.update({ + Index(table, (table.key, Index.Equality())), + Index( + table, + (table.user, Index.Equality()), + (table.state, Index.Equality())), + Index( + table, + (table.key, Index.Equality()), + (table.application, Index.Equality()), + (table.state, Index.Equality())), + }) + @classmethod def default_key(cls): return ''.join(uuid.uuid4().hex for _ in range(4)) diff --git a/trytond/tests/mptt.py b/trytond/tests/mptt.py index 704c361ed0..d9ac62a606 100644 --- a/trytond/tests/mptt.py +++ b/trytond/tests/mptt.py @@ -12,10 +12,9 @@ class MPTT(DeactivableMixin, tree(), ModelSQL, ModelView): 'Modified Preorder Tree Traversal' __name__ = 'test.mptt' name = fields.Char('Name', required=True) - parent = fields.Many2One('test.mptt', "Parent", select=True, - left="left", right="right") - left = fields.Integer('Left', required=True, select=True) - right = fields.Integer('Right', required=True, select=True) + parent = fields.Many2One('test.mptt', "Parent", left="left", right="right") + left = fields.Integer('Left', required=True) + right = fields.Integer('Right', required=True) childs = fields.One2Many('test.mptt', 'parent', 'Children') @staticmethod diff --git a/trytond/tests/path.py b/trytond/tests/path.py index 1d1d2c55d7..9e18427aec 100644 --- a/trytond/tests/path.py +++ b/trytond/tests/path.py @@ -8,7 +8,7 @@ class Path(DeactivableMixin, tree(), ModelSQL): "Path" __name__ = 'test.path' name = fields.Char("Name", required=True) - parent = fields.Many2One('test.path', "Parent", select=True, path='path') + parent = fields.Many2One('test.path', "Parent", path='path') path = fields.Char("Path") children = fields.One2Many('test.path', 'parent', "Children") diff --git a/trytond/tests/test_model_index.py b/trytond/tests/test_model_index.py new file mode 100644 index 0000000000..7dac68a6e2 --- /dev/null +++ b/trytond/tests/test_model_index.py @@ -0,0 +1,140 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of this +# repository contains the full copyright notices and license terms. + +import unittest + +from sql import Table + +from trytond.model import Index + + +class ModelIndexTestCase(unittest.TestCase): + "Test Model Index" + + def test_index_equality(self): + "Test Index equality" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality(order='DESC')), + (table1.amount, Index.Range()), + where=table1.name == 'foo') + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality(order='DESC')), + (table2.amount, Index.Range()), + where=table2.name == 'foo') + + self.assertEqual(index1, index2) + + def test_index_inequality_table(self): + "Test Index inequality on table" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality())) + table2 = Table('foo') + index2 = Index( + table2, + (table2.name, Index.Equality())) + + self.assertNotEqual(index1, index2) + + def test_index_inequality_expression(self): + "Test Index inequality on usage" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality()), + (table1.amount, Index.Range())) + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality()), + (table2.value, Index.Range())) + + self.assertNotEqual(index1, index2) + + def test_index_inequality_usage(self): + "Test Index inequality on usage" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality()), + (table1.amount, Index.Range())) + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality()), + (table2.amount, Index.Similarity())) + + self.assertNotEqual(index1, index2) + + def test_index_inequality_usage_option(self): + "Test Index inequality on usage option" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality(order='DESC')), + (table1.amount, Index.Range()), + where=table1.name == 'foo') + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality(order='ASC')), + (table2.amount, Index.Range()), + where=table2.name == 'foo') + + self.assertNotEqual(index1, index2) + + def test_index_inequality_option_expression(self): + "Test Index inequality on option param" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality(order='DESC')), + (table1.amount, Index.Range()), + where=table1.name == 'foo') + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality(order='DESC')), + (table2.amount, Index.Range()), + where=table2.amount == 'foo') + + self.assertEqual(index1, index2) + + def test_index_inequality_option_param(self): + "Test Index inequality on option param" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality(order='DESC')), + (table1.amount, Index.Range()), + where=table1.name == 'foo') + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality(order='DESC')), + (table2.amount, Index.Range()), + where=table2.name == 'bar') + + self.assertEqual(index1, index2) + + def test_index_hash(self): + "Test Index hash" + table1 = Table('test') + index1 = Index( + table1, + (table1.name, Index.Equality(order='DESC')), + (table1.amount, Index.Range()), + where=table1.name == 'foo') + table2 = Table('test') + index2 = Index( + table2, + (table2.name, Index.Equality(order='DESC')), + (table2.amount, Index.Range()), + where=table2.name == 'foo') + + self.assertEqual(hash(index1), hash(index2))