From ca61f7cc7bd15e68669d3073fe8b23beddb842f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Thu, 10 Oct 2024 00:34:53 +0200 Subject: [PATCH] Support range functions and operators in EXCLUDE constraint Closes #7471 --- trytond/CHANGELOG | 1 + trytond/doc/ref/index.rst | 1 + trytond/doc/ref/sql/functions.rst | 18 +++ trytond/doc/ref/sql/index.rst | 14 +++ trytond/doc/ref/sql/operators.rst | 18 +++ trytond/trytond/backend/database.py | 3 + .../trytond/backend/postgresql/database.py | 14 +++ trytond/trytond/backend/postgresql/table.py | 44 +++++-- trytond/trytond/model/modelsql.py | 35 +++++- trytond/trytond/sql/__init__.py | 2 + trytond/trytond/sql/functions.py | 29 +++++ trytond/trytond/sql/operators.py | 116 ++++++++++++++++++ trytond/trytond/tests/test_backend.py | 79 +++++++++++- 13 files changed, 356 insertions(+), 18 deletions(-) create mode 100644 trytond/doc/ref/sql/functions.rst create mode 100644 trytond/doc/ref/sql/index.rst create mode 100644 trytond/doc/ref/sql/operators.rst create mode 100644 trytond/trytond/sql/__init__.py create mode 100644 trytond/trytond/sql/functions.py create mode 100644 trytond/trytond/sql/operators.py diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG index 1a48e77b2e..46f6a2d9bd 100644 --- a/trytond/CHANGELOG +++ b/trytond/CHANGELOG @@ -1,3 +1,4 @@ +* Support range functions and operators in EXCLUDE constraint * Add sparkline chart tools * Add setup methods to activate_modules * Add method to the backend to estimate number of rows diff --git a/trytond/doc/ref/index.rst b/trytond/doc/ref/index.rst index 10a76f6500..ee1fd1cfd9 100644 --- a/trytond/doc/ref/index.rst +++ b/trytond/doc/ref/index.rst @@ -17,6 +17,7 @@ Reference of the ``trytond`` API: transaction exceptions backend + sql/index tools/index pool rpc diff --git a/trytond/doc/ref/sql/functions.rst b/trytond/doc/ref/sql/functions.rst new file mode 100644 index 0000000000..6d425d3057 --- /dev/null +++ b/trytond/doc/ref/sql/functions.rst @@ -0,0 +1,18 @@ +.. _ref-sql-functions: +.. module:: trytond.sql.functions + +SQL Functions +============= + +.. class:: NumRange(lower, upper[, bounds]) + + A range of numeric. + ``bounds`` can be ``()``, ``(]``, ``[)`` (default) or ``[]`` + +.. class:: TSRange(lower, upper[, bounds]) + + A range of timestamp. + +.. class:: DateRange(lower, upper[, bounds]) + + A range of date. diff --git a/trytond/doc/ref/sql/index.rst b/trytond/doc/ref/sql/index.rst new file mode 100644 index 0000000000..1e97279310 --- /dev/null +++ b/trytond/doc/ref/sql/index.rst @@ -0,0 +1,14 @@ +.. _ref-sql-index: + +=== +SQL +=== + +Custom SQL constructors. + + +.. toctree:: + :maxdepth: 1 + + functions + operators diff --git a/trytond/doc/ref/sql/operators.rst b/trytond/doc/ref/sql/operators.rst new file mode 100644 index 0000000000..387f5ad743 --- /dev/null +++ b/trytond/doc/ref/sql/operators.rst @@ -0,0 +1,18 @@ +.. _ref-sql-operators: +.. module:: trytond.sql.operators + +SQL Operators +============= + + +.. class:: RangeOperator(first, second) + + Does the first range contain the second range or element? + +.. class:: RangeIn(first, second) + + Is the first range or element contained in the second range? + +.. class:: RangeOverlap(first, second) + + Does the ranges overlap, that is, have any elements in common? diff --git a/trytond/trytond/backend/database.py b/trytond/trytond/backend/database.py index 575bf33e08..c19de03740 100644 --- a/trytond/trytond/backend/database.py +++ b/trytond/trytond/backend/database.py @@ -79,6 +79,9 @@ class DatabaseInterface(object): def has_constraint(self, constraint): raise NotImplementedError + def has_range(self): + return False + def has_returning(self): return False diff --git a/trytond/trytond/backend/postgresql/database.py b/trytond/trytond/backend/postgresql/database.py index c99b14ec7d..46c6cbf493 100644 --- a/trytond/trytond/backend/postgresql/database.py +++ b/trytond/trytond/backend/postgresql/database.py @@ -17,6 +17,7 @@ from psycopg2.extensions import ( register_type) from psycopg2.pool import PoolError, ThreadedConnectionPool from psycopg2.sql import SQL, Identifier +from sql.operators import NotEqual try: from psycopg2.extensions import PYDATE, PYDATETIME, PYINTERVAL, PYTIME @@ -37,8 +38,11 @@ from sql.operators import BinaryOperator, Concat from trytond.backend.database import DatabaseInterface, SQLType from trytond.config import config, parse_uri +from trytond.sql.operators import RangeOperator from trytond.tools.gevent import is_gevent_monkey_patched +from .table import index_method + __all__ = [ 'Database', 'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError', @@ -478,6 +482,16 @@ class Database(DatabaseInterface): return AdvisoryLock(id) def has_constraint(self, constraint): + from trytond.model.modelsql import Exclude + if isinstance(constraint, Exclude): + if (index_method(constraint) == 'GIST' + and any(not issubclass(o, (RangeOperator, NotEqual)) + for o in constraint.operators) + and not self.has_extension('btree_gist')): + return False + return True + + def has_range(self): return True def has_multirow_insert(self): diff --git a/trytond/trytond/backend/postgresql/table.py b/trytond/trytond/backend/postgresql/table.py index f172934717..bb3aec41ac 100644 --- a/trytond/trytond/backend/postgresql/table.py +++ b/trytond/trytond/backend/postgresql/table.py @@ -4,9 +4,11 @@ import logging import re from psycopg2.sql import SQL, Identifier +from sql.operators import NotEqual from trytond.backend.table import ( IndexTranslatorInterface, TableHandlerInterface) +from trytond.sql.operators import RangeOperator from trytond.transaction import Transaction __all__ = ['TableHandler'] @@ -494,18 +496,27 @@ class TableHandler(TableHandlerInterface): raise Exception('Not null action not supported!') def add_constraint(self, ident, constraint): - ident = self.convert_name(self.table_name + "_" + ident) - if ident in self._constraints: - # This constrain already exist - return - cursor = Transaction().connection.cursor() - cursor.execute( - SQL('ALTER TABLE {} ADD CONSTRAINT {} {}').format( - Identifier(self.table_name), - Identifier(ident), - SQL(str(constraint))), - constraint.params) - self._update_definitions(constraints=True) + from trytond.model.modelsql import Exclude + transaction = Transaction() + database = transaction.database + + if database.has_constraint(constraint): + ident = self.convert_name(self.table_name + "_" + ident) + if ident in self._constraints: + # This constrain already exist + return + cursor = Transaction().connection.cursor() + if isinstance(constraint, Exclude): + definition = constraint.__str__(using=index_method(constraint)) + else: + definition = str(constraint) + cursor.execute( + SQL('ALTER TABLE {} ADD CONSTRAINT {} {}').format( + Identifier(self.table_name), + Identifier(ident), + SQL(definition)), + constraint.params) + self._update_definitions(constraints=True) def drop_constraint(self, ident, table=None): ident = self.convert_name((table or self.table_name) + "_" + ident) @@ -730,3 +741,12 @@ class TrigramTranslator(IndexMixin, IndexTranslatorInterface): if usage.__class__.__name__ == 'Similarity': params['opclass'] = SQL('gin_trgm_ops') return params + + +def index_method(exclude): + if any(issubclass(o, (RangeOperator, NotEqual)) + for o in exclude.operators): + method = 'GIST' + else: + method = 'BTREE' + return method diff --git a/trytond/trytond/model/modelsql.py b/trytond/trytond/model/modelsql.py index d4f41345f5..a54d9b34b5 100644 --- a/trytond/trytond/model/modelsql.py +++ b/trytond/trytond/model/modelsql.py @@ -21,6 +21,7 @@ from trytond.i18n import gettext from trytond.pool import Pool from trytond.pyson import PYSONDecoder, PYSONEncoder from trytond.rpc import RPC +from trytond.sql.functions import Range from trytond.tools import cursor_dict, grouped_slice, reduce_ids from trytond.tools.domain_inversion import simplify from trytond.transaction import ( @@ -137,13 +138,15 @@ class Exclude(Constraint): def where(self): return self._where - def __str__(self): + def __str__(self, using=''): exclude = ', '.join('%s WITH %s' % (column, operator._operator) for column, operator in self.excludes) where = '' if self.where: where = ' WHERE ' + str(self.where) - return 'EXCLUDE (%s)' % exclude + where + if using: + using = 'USING ' + using + return 'EXCLUDE %s (%s)' % (using, exclude) + where @property def params(self): @@ -2207,7 +2210,17 @@ class ModelSQL(ModelStorage): table = sql.table if isinstance(sql, (Unique, Exclude)): lock(connection, cls._table) - columns = list(sql.columns) + if not database.has_range(): + columns = [] + for col in sql.columns: + if isinstance(col, Range): + columns.extend( + a if isinstance(a, Expression) else Literal(a) + for a in col.args) + else: + columns.append(col) + else: + columns = list(sql.columns) columns.insert(0, table.id) in_max = transaction.database.IN_MAX // (len(columns) + 1) for sub_ids in grouped_slice(ids, in_max): @@ -2219,9 +2232,21 @@ class ModelSQL(ModelStorage): where = Literal(False) for row in cursor: - clause = table.id != row[0] + row = list(row) + clause = table.id != row.pop(0) + if not database.has_range(): + values = [] + for col in sql.columns: + if isinstance(col, Range): + range_ = col.__class__( + row.pop(0), row.pop(0), row.pop(0)) + values.append(range_) + else: + values.append(row.pop(0)) + else: + values = row for column, operator, value in zip( - sql.columns, sql.operators, row[1:]): + sql.columns, sql.operators, values): if value is None: # NULL is always unique clause &= Literal(False) diff --git a/trytond/trytond/sql/__init__.py b/trytond/trytond/sql/__init__.py new file mode 100644 index 0000000000..4effdfa4a2 --- /dev/null +++ b/trytond/trytond/sql/__init__.py @@ -0,0 +1,2 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. diff --git a/trytond/trytond/sql/functions.py b/trytond/trytond/sql/functions.py new file mode 100644 index 0000000000..641aa28fa5 --- /dev/null +++ b/trytond/trytond/sql/functions.py @@ -0,0 +1,29 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. + +from sql.functions import Function + +__all__ = ['NumRange', 'TSRange', 'DateRange'] + + +class Range(Function): + __slots__ = () + + def __init__(self, lower, upper, bounds='[)'): + assert bounds in {'()', '(]', '[)', '[]'} + super().__init__(lower, upper, bounds) + + +class NumRange(Range): + __slots__ = () + _function = 'NUMRANGE' + + +class TSRange(Range): + __slots__ = () + _function = 'TSRANGE' + + +class DateRange(Range): + __slots__ = () + _function = 'DATERANGE' diff --git a/trytond/trytond/sql/operators.py b/trytond/trytond/sql/operators.py new file mode 100644 index 0000000000..409ca6c91b --- /dev/null +++ b/trytond/trytond/sql/operators.py @@ -0,0 +1,116 @@ +# This file is part of Tryton. The COPYRIGHT file at the top level of +# this repository contains the full copyright notices and license terms. + +from sql import Expression, Literal, Null +from sql.conditionals import Case +from sql.operators import BinaryOperator + +from trytond.transaction import Transaction + +from .functions import Range + +__all__ = ['RangeContain', 'RangeIn', 'RangeOverlap'] + + +def _has_range(): + return Transaction().database.has_range() + + +class RangeOperator(BinaryOperator): + __slots__ = () + + def __str__(self): + if not _has_range(): + return str(self._sql_expression) + return super().__str__() + + @property + def params(self): + if not _has_range(): + return self._sql_expression.params + return super().params + + +class RangeContain(RangeOperator): + __slots__ = () + _operator = '@>' + + @property + def _sql_expression(self): + lower1, upper1, bounds1 = self.left.args + if isinstance(self.right, Range): + lower2, upper2, bounds2 = self.right.args + else: + lower2 = upper2 = self.right + bounds2 = '[]' + + if not isinstance(lower1, Expression): + lower1 = Literal(lower1) + if not isinstance(upper1, Expression): + upper1 = Literal(upper1) + if not isinstance(lower2, Expression): + lower2 = Literal(lower2) + if not isinstance(upper2, Expression): + upper2 = Literal(upper2) + + if bounds1[0] == '(' and bounds2[0] == '[': + expression1 = lower1 < lower2 + else: + expression1 = lower1 <= lower2 + expression1 = Case( + ((lower1 == Null), True), + ((lower2 == Null), False), + else_=expression1) + if bounds1[1] == ')' and bounds2[1] == ']': + expression2 = upper1 > upper2 + else: + expression2 = upper1 >= upper2 + expression2 = Case( + ((upper1 == Null), True), + ((upper2 == Null), False), + else_=expression2) + return (expression1) & (expression2) + + +class RangeIn(RangeOperator): + __slots__ = () + _operator = '<@' + + @property + def _sql_expression(self): + return RangeContain(self.right, self.left)._sql_expression + + +class RangeOverlap(RangeOperator): + __slots__ = () + _operator = '&&' + + @property + def _sql_expression(self): + lower1, upper1, bounds1 = self.left.args + lower2, upper2, bounds2 = self.right.args + + if not isinstance(lower1, Expression): + lower1 = Literal(lower1) + if not isinstance(upper1, Expression): + upper1 = Literal(upper1) + if not isinstance(lower2, Expression): + lower2 = Literal(lower2) + if not isinstance(upper2, Expression): + upper2 = Literal(upper2) + + if bounds1[0] == '[' and bounds2[1] == ']': + expression1 = lower1 <= upper2 + else: + expression1 = lower1 < upper2 + expression1 = Case( + ((lower1 == Null) | (upper2 == Null), True), + else_=expression1) + if bounds1[1] == ']' and bounds2[0] == '[': + expression2 = lower2 <= upper1 + else: + expression2 = lower2 < upper1 + expression2 = Case( + ((lower2 == Null) | (upper1 == Null), True), + else_=expression2) + return (expression1) & (expression2) diff --git a/trytond/trytond/tests/test_backend.py b/trytond/trytond/tests/test_backend.py index a83887f434..2b55beb5d7 100644 --- a/trytond/trytond/tests/test_backend.py +++ b/trytond/trytond/tests/test_backend.py @@ -3,10 +3,12 @@ import datetime as dt import math -from sql import Literal, Select, Table, functions +from sql import Cast, Literal, Select, Table, functions from sql.functions import CurrentTimestamp, DateTrunc, ToChar from trytond.model import fields +from trytond.sql.functions import NumRange +from trytond.sql.operators import RangeContain, RangeIn, RangeOverlap from trytond.tests.test_tryton import ( TestCase, activate_module, with_transaction) from trytond.transaction import Transaction @@ -183,6 +185,81 @@ class BackendTestCase(TestCase): value, = cursor.fetchone() self.assertEqual(value, None) + @with_transaction() + def test_operator_range_contain(self): + "Test Range Contain operator" + cursor = Transaction().connection.cursor() + numeric = fields.Numeric("Test") + + def Num(value): + return Cast(value, numeric.sql_type().base) + + for expression, value in [ + (RangeContain(NumRange(2, 4), NumRange(2, 3)), True), + (RangeContain(NumRange(2, 4), NumRange(1, 2)), False), + (RangeContain( + NumRange(2, 4, '(]'), NumRange(2, 3, '[)')), False), + (RangeContain(NumRange(None, 4), NumRange(2, 3)), True), + (RangeContain(NumRange(None, 2), NumRange(2, 3)), False), + (RangeContain(NumRange(None, 2), NumRange(None, 2)), True), + (RangeContain(NumRange(2, 4), NumRange(2, None)), False), + (RangeContain(NumRange(2, None), NumRange(2, None)), True), + (RangeContain(NumRange(2, 4), Num(1)), False), + (RangeContain(NumRange(2, 4), Num(3)), True), + (RangeContain(NumRange(2, 4), Num(2)), True), + (RangeContain(NumRange(2, 4), Num(4)), False), + ]: + with self.subTest(expression=expression.params): + cursor.execute(*Select([expression])) + result, = cursor.fetchone() + self.assertEqual(result, value) + + @with_transaction() + def test_operator_range_in(self): + "Test Range In operator" + cursor = Transaction().connection.cursor() + numeric = fields.Numeric("Test") + + def Num(value): + return Cast(value, numeric.sql_type().base) + + for expression, value in [ + (RangeIn(NumRange(2, 3), NumRange(2, 4)), True), + (RangeIn(Num(2), NumRange(2, 4)), True), + ]: + with self.subTest(expression=expression.params): + cursor.execute(*Select([expression])) + result, = cursor.fetchone() + self.assertEqual(result, value) + + @with_transaction() + def test_operator_range_overlap(self): + "test Range Overlap operator" + cursor = Transaction().connection.cursor() + + for expression, value in [ + (RangeOverlap(NumRange(3, 7), NumRange(4, 12)), True), + (RangeOverlap(NumRange(3, 4), NumRange(7, 12)), False), + (RangeOverlap(NumRange(2, 3), NumRange(3, 4)), False), + (RangeOverlap( + NumRange(2, 3, '(]'), NumRange(3, 4, '[)')), True), + (RangeOverlap(NumRange(None, 3), NumRange(3, 4)), False), + (RangeOverlap(NumRange(2, None), NumRange(3, 4)), True), + (RangeOverlap(NumRange(None, 3), NumRange(3, None)), False), + (RangeOverlap(NumRange(None, 3), NumRange(None, 4)), True), + (RangeOverlap(NumRange(2, None), NumRange(3, None)), True), + ]: + with self.subTest(expression=expression.params): + cursor.execute(*Select([expression])) + result, = cursor.fetchone() + self.assertEqual(result, value) + + expression.left, expression.right = ( + expression.right, expression.left) + cursor.execute(*Select([expression])) + result, = cursor.fetchone() + self.assertEqual(result, value) + @with_transaction() def test_estimated_count(self): "Test estimated count queries"