Support range functions and operators in EXCLUDE constraint

Closes #7471

trytond/CHANGELOG

@@ -1,3 +1,4 @@
+* Support range functions and operators in EXCLUDE constraint
 * Add sparkline chart tools
 * Add setup methods to activate_modules
 * Add method to the backend to estimate number of rows

trytond/doc/ref/index.rst

@@ -17,6 +17,7 @@ Reference of the ``trytond`` API:
    transaction
    exceptions
    backend
+   sql/index
    tools/index
    pool
    rpc

trytond/doc/ref/sql/functions.rst

@@ -0,0 +1,18 @@
+.. _ref-sql-functions:
+.. module:: trytond.sql.functions
+
+SQL Functions
+=============
+
+.. class:: NumRange(lower, upper[, bounds])
+
+   A range of numeric.
+   ``bounds`` can be ``()``, ``(]``, ``[)`` (default) or ``[]``
+
+.. class:: TSRange(lower, upper[, bounds])
+
+   A range of timestamp.
+
+.. class:: DateRange(lower, upper[, bounds])
+
+   A range of date.

trytond/doc/ref/sql/index.rst

@@ -0,0 +1,14 @@
+.. _ref-sql-index:
+
+===
+SQL
+===
+
+Custom SQL constructors.
+
+
+.. toctree::
+   :maxdepth: 1
+
+   functions
+   operators

trytond/doc/ref/sql/operators.rst

@@ -0,0 +1,18 @@
+.. _ref-sql-operators:
+.. module:: trytond.sql.operators
+
+SQL Operators
+=============
+
+
+.. class:: RangeOperator(first, second)
+
+   Does the first range contain the second range or element?
+
+.. class:: RangeIn(first, second)
+
+   Is the first range or element contained in the second range?
+
+.. class:: RangeOverlap(first, second)
+
+   Does the ranges overlap, that is, have any elements in common?

trytond/trytond/backend/database.py

@@ -79,6 +79,9 @@ class DatabaseInterface(object):
     def has_constraint(self, constraint):
         raise NotImplementedError
 
+    def has_range(self):
+        return False
+
     def has_returning(self):
         return False
 

trytond/trytond/backend/postgresql/database.py

@@ -17,6 +17,7 @@ from psycopg2.extensions import (
     register_type)
 from psycopg2.pool import PoolError, ThreadedConnectionPool
 from psycopg2.sql import SQL, Identifier
+from sql.operators import NotEqual
 
 try:
     from psycopg2.extensions import PYDATE, PYDATETIME, PYINTERVAL, PYTIME
@@ -37,8 +38,11 @@ from sql.operators import BinaryOperator, Concat
 
 from trytond.backend.database import DatabaseInterface, SQLType
 from trytond.config import config, parse_uri
+from trytond.sql.operators import RangeOperator
 from trytond.tools.gevent import is_gevent_monkey_patched
 
+from .table import index_method
+
 __all__ = [
     'Database',
     'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError',
@@ -478,6 +482,16 @@ class Database(DatabaseInterface):
             return AdvisoryLock(id)
 
     def has_constraint(self, constraint):
+        from trytond.model.modelsql import Exclude
+        if isinstance(constraint, Exclude):
+            if (index_method(constraint) == 'GIST'
+                    and any(not issubclass(o, (RangeOperator, NotEqual))
+                        for o in constraint.operators)
+                    and not self.has_extension('btree_gist')):
+                return False
+        return True
+
+    def has_range(self):
         return True
 
     def has_multirow_insert(self):

trytond/trytond/backend/postgresql/table.py

@@ -4,9 +4,11 @@ import logging
 import re
 
 from psycopg2.sql import SQL, Identifier
+from sql.operators import NotEqual
 
 from trytond.backend.table import (
     IndexTranslatorInterface, TableHandlerInterface)
+from trytond.sql.operators import RangeOperator
 from trytond.transaction import Transaction
 
 __all__ = ['TableHandler']
@@ -494,18 +496,27 @@ class TableHandler(TableHandlerInterface):
                 raise Exception('Not null action not supported!')
 
     def add_constraint(self, ident, constraint):
-        ident = self.convert_name(self.table_name + "_" + ident)
-        if ident in self._constraints:
-            # This constrain already exist
-            return
-        cursor = Transaction().connection.cursor()
-        cursor.execute(
-            SQL('ALTER TABLE {} ADD CONSTRAINT {} {}').format(
-                Identifier(self.table_name),
-                Identifier(ident),
-                SQL(str(constraint))),
-            constraint.params)
-        self._update_definitions(constraints=True)
+        from trytond.model.modelsql import Exclude
+        transaction = Transaction()
+        database = transaction.database
+
+        if database.has_constraint(constraint):
+            ident = self.convert_name(self.table_name + "_" + ident)
+            if ident in self._constraints:
+                # This constrain already exist
+                return
+            cursor = Transaction().connection.cursor()
+            if isinstance(constraint, Exclude):
+                definition = constraint.__str__(using=index_method(constraint))
+            else:
+                definition = str(constraint)
+            cursor.execute(
+                SQL('ALTER TABLE {} ADD CONSTRAINT {} {}').format(
+                    Identifier(self.table_name),
+                    Identifier(ident),
+                    SQL(definition)),
+                constraint.params)
+            self._update_definitions(constraints=True)
 
     def drop_constraint(self, ident, table=None):
         ident = self.convert_name((table or self.table_name) + "_" + ident)
@@ -730,3 +741,12 @@ class TrigramTranslator(IndexMixin, IndexTranslatorInterface):
         if usage.__class__.__name__ == 'Similarity':
             params['opclass'] = SQL('gin_trgm_ops')
         return params
+
+
+def index_method(exclude):
+    if any(issubclass(o, (RangeOperator, NotEqual))
+            for o in exclude.operators):
+        method = 'GIST'
+    else:
+        method = 'BTREE'
+    return method

trytond/trytond/model/modelsql.py

@@ -21,6 +21,7 @@ from trytond.i18n import gettext
 from trytond.pool import Pool
 from trytond.pyson import PYSONDecoder, PYSONEncoder
 from trytond.rpc import RPC
+from trytond.sql.functions import Range
 from trytond.tools import cursor_dict, grouped_slice, reduce_ids
 from trytond.tools.domain_inversion import simplify
 from trytond.transaction import (
@@ -137,13 +138,15 @@ class Exclude(Constraint):
     def where(self):
         return self._where
 
-    def __str__(self):
+    def __str__(self, using=''):
         exclude = ', '.join('%s WITH %s' % (column, operator._operator)
             for column, operator in self.excludes)
         where = ''
         if self.where:
             where = ' WHERE ' + str(self.where)
-        return 'EXCLUDE (%s)' % exclude + where
+        if using:
+            using = 'USING ' + using
+        return 'EXCLUDE %s (%s)' % (using, exclude) + where
 
     @property
     def params(self):
@@ -2207,7 +2210,17 @@ class ModelSQL(ModelStorage):
             table = sql.table
             if isinstance(sql, (Unique, Exclude)):
                 lock(connection, cls._table)
-                columns = list(sql.columns)
+                if not database.has_range():
+                    columns = []
+                    for col in sql.columns:
+                        if isinstance(col, Range):
+                            columns.extend(
+                                a if isinstance(a, Expression) else Literal(a)
+                                for a in col.args)
+                        else:
+                            columns.append(col)
+                else:
+                    columns = list(sql.columns)
                 columns.insert(0, table.id)
                 in_max = transaction.database.IN_MAX // (len(columns) + 1)
                 for sub_ids in grouped_slice(ids, in_max):
@@ -2219,9 +2232,21 @@ class ModelSQL(ModelStorage):
 
                     where = Literal(False)
                     for row in cursor:
-                        clause = table.id != row[0]
+                        row = list(row)
+                        clause = table.id != row.pop(0)
+                        if not database.has_range():
+                            values = []
+                            for col in sql.columns:
+                                if isinstance(col, Range):
+                                    range_ = col.__class__(
+                                        row.pop(0), row.pop(0), row.pop(0))
+                                    values.append(range_)
+                                else:
+                                    values.append(row.pop(0))
+                        else:
+                            values = row
                         for column, operator, value in zip(
-                                sql.columns, sql.operators, row[1:]):
+                                sql.columns, sql.operators, values):
                             if value is None:
                                 # NULL is always unique
                                 clause &= Literal(False)

trytond/trytond/sql/__init__.py

@@ -0,0 +1,2 @@
+# This file is part of Tryton.  The COPYRIGHT file at the top level of
+# this repository contains the full copyright notices and license terms.

trytond/trytond/sql/functions.py

@@ -0,0 +1,29 @@
+# This file is part of Tryton.  The COPYRIGHT file at the top level of
+# this repository contains the full copyright notices and license terms.
+
+from sql.functions import Function
+
+__all__ = ['NumRange', 'TSRange', 'DateRange']
+
+
+class Range(Function):
+    __slots__ = ()
+
+    def __init__(self, lower, upper, bounds='[)'):
+        assert bounds in {'()', '(]', '[)', '[]'}
+        super().__init__(lower, upper, bounds)
+
+
+class NumRange(Range):
+    __slots__ = ()
+    _function = 'NUMRANGE'
+
+
+class TSRange(Range):
+    __slots__ = ()
+    _function = 'TSRANGE'
+
+
+class DateRange(Range):
+    __slots__ = ()
+    _function = 'DATERANGE'

trytond/trytond/sql/operators.py

@@ -0,0 +1,116 @@
+# This file is part of Tryton.  The COPYRIGHT file at the top level of
+# this repository contains the full copyright notices and license terms.
+
+from sql import Expression, Literal, Null
+from sql.conditionals import Case
+from sql.operators import BinaryOperator
+
+from trytond.transaction import Transaction
+
+from .functions import Range
+
+__all__ = ['RangeContain', 'RangeIn', 'RangeOverlap']
+
+
+def _has_range():
+    return Transaction().database.has_range()
+
+
+class RangeOperator(BinaryOperator):
+    __slots__ = ()
+
+    def __str__(self):
+        if not _has_range():
+            return str(self._sql_expression)
+        return super().__str__()
+
+    @property
+    def params(self):
+        if not _has_range():
+            return self._sql_expression.params
+        return super().params
+
+
+class RangeContain(RangeOperator):
+    __slots__ = ()
+    _operator = '@>'
+
+    @property
+    def _sql_expression(self):
+        lower1, upper1, bounds1 = self.left.args
+        if isinstance(self.right, Range):
+            lower2, upper2, bounds2 = self.right.args
+        else:
+            lower2 = upper2 = self.right
+            bounds2 = '[]'
+
+        if not isinstance(lower1, Expression):
+            lower1 = Literal(lower1)
+        if not isinstance(upper1, Expression):
+            upper1 = Literal(upper1)
+        if not isinstance(lower2, Expression):
+            lower2 = Literal(lower2)
+        if not isinstance(upper2, Expression):
+            upper2 = Literal(upper2)
+
+        if bounds1[0] == '(' and bounds2[0] == '[':
+            expression1 = lower1 < lower2
+        else:
+            expression1 = lower1 <= lower2
+        expression1 = Case(
+            ((lower1 == Null), True),
+            ((lower2 == Null), False),
+            else_=expression1)
+        if bounds1[1] == ')' and bounds2[1] == ']':
+            expression2 = upper1 > upper2
+        else:
+            expression2 = upper1 >= upper2
+        expression2 = Case(
+            ((upper1 == Null), True),
+            ((upper2 == Null), False),
+            else_=expression2)
+        return (expression1) & (expression2)
+
+
+class RangeIn(RangeOperator):
+    __slots__ = ()
+    _operator = '<@'
+
+    @property
+    def _sql_expression(self):
+        return RangeContain(self.right, self.left)._sql_expression
+
+
+class RangeOverlap(RangeOperator):
+    __slots__ = ()
+    _operator = '&&'
+
+    @property
+    def _sql_expression(self):
+        lower1, upper1, bounds1 = self.left.args
+        lower2, upper2, bounds2 = self.right.args
+
+        if not isinstance(lower1, Expression):
+            lower1 = Literal(lower1)
+        if not isinstance(upper1, Expression):
+            upper1 = Literal(upper1)
+        if not isinstance(lower2, Expression):
+            lower2 = Literal(lower2)
+        if not isinstance(upper2, Expression):
+            upper2 = Literal(upper2)
+
+        if bounds1[0] == '[' and bounds2[1] == ']':
+            expression1 = lower1 <= upper2
+        else:
+            expression1 = lower1 < upper2
+        expression1 = Case(
+            ((lower1 == Null) | (upper2 == Null), True),
+            else_=expression1)
+        if bounds1[1] == ']' and bounds2[0] == '[':
+            expression2 = lower2 <= upper1
+        else:
+            expression2 = lower2 < upper1
+        expression2 = Case(
+            ((lower2 == Null) | (upper1 == Null), True),
+            else_=expression2)
+        return (expression1) & (expression2)

trytond/trytond/tests/test_backend.py

@@ -3,10 +3,12 @@
 import datetime as dt
 import math
 
-from sql import Literal, Select, Table, functions
+from sql import Cast, Literal, Select, Table, functions
 from sql.functions import CurrentTimestamp, DateTrunc, ToChar
 
 from trytond.model import fields
+from trytond.sql.functions import NumRange
+from trytond.sql.operators import RangeContain, RangeIn, RangeOverlap
 from trytond.tests.test_tryton import (
     TestCase, activate_module, with_transaction)
 from trytond.transaction import Transaction
@@ -183,6 +185,81 @@ class BackendTestCase(TestCase):
         value, = cursor.fetchone()
         self.assertEqual(value, None)
 
+    @with_transaction()
+    def test_operator_range_contain(self):
+        "Test Range Contain operator"
+        cursor = Transaction().connection.cursor()
+        numeric = fields.Numeric("Test")
+
+        def Num(value):
+            return Cast(value, numeric.sql_type().base)
+
+        for expression, value in [
+                (RangeContain(NumRange(2, 4), NumRange(2, 3)), True),
+                (RangeContain(NumRange(2, 4), NumRange(1, 2)), False),
+                (RangeContain(
+                        NumRange(2, 4, '(]'), NumRange(2, 3, '[)')), False),
+                (RangeContain(NumRange(None, 4), NumRange(2, 3)), True),
+                (RangeContain(NumRange(None, 2), NumRange(2, 3)), False),
+                (RangeContain(NumRange(None, 2), NumRange(None, 2)), True),
+                (RangeContain(NumRange(2, 4), NumRange(2, None)), False),
+                (RangeContain(NumRange(2, None), NumRange(2, None)), True),
+                (RangeContain(NumRange(2, 4), Num(1)), False),
+                (RangeContain(NumRange(2, 4), Num(3)), True),
+                (RangeContain(NumRange(2, 4), Num(2)), True),
+                (RangeContain(NumRange(2, 4), Num(4)), False),
+                ]:
+            with self.subTest(expression=expression.params):
+                cursor.execute(*Select([expression]))
+                result, = cursor.fetchone()
+                self.assertEqual(result, value)
+
+    @with_transaction()
+    def test_operator_range_in(self):
+        "Test Range In operator"
+        cursor = Transaction().connection.cursor()
+        numeric = fields.Numeric("Test")
+
+        def Num(value):
+            return Cast(value, numeric.sql_type().base)
+
+        for expression, value in [
+                (RangeIn(NumRange(2, 3), NumRange(2, 4)), True),
+                (RangeIn(Num(2), NumRange(2, 4)), True),
+                ]:
+            with self.subTest(expression=expression.params):
+                cursor.execute(*Select([expression]))
+                result, = cursor.fetchone()
+                self.assertEqual(result, value)
+
+    @with_transaction()
+    def test_operator_range_overlap(self):
+        "test Range Overlap operator"
+        cursor = Transaction().connection.cursor()
+
+        for expression, value in [
+                (RangeOverlap(NumRange(3, 7), NumRange(4, 12)), True),
+                (RangeOverlap(NumRange(3, 4), NumRange(7, 12)), False),
+                (RangeOverlap(NumRange(2, 3), NumRange(3, 4)), False),
+                (RangeOverlap(
+                        NumRange(2, 3, '(]'), NumRange(3, 4, '[)')), True),
+                (RangeOverlap(NumRange(None, 3), NumRange(3, 4)), False),
+                (RangeOverlap(NumRange(2, None), NumRange(3, 4)), True),
+                (RangeOverlap(NumRange(None, 3), NumRange(3, None)), False),
+                (RangeOverlap(NumRange(None, 3), NumRange(None, 4)), True),
+                (RangeOverlap(NumRange(2, None), NumRange(3, None)), True),
+                ]:
+            with self.subTest(expression=expression.params):
+                cursor.execute(*Select([expression]))
+                result, = cursor.fetchone()
+                self.assertEqual(result, value)
+
+                expression.left, expression.right = (
+                    expression.right, expression.left)
+                cursor.execute(*Select([expression]))
+                result, = cursor.fetchone()
+                self.assertEqual(result, value)
+
     @with_transaction()
     def test_estimated_count(self):
         "Test estimated count queries"