From 45d406e27072f37660307dd8cf69a5feb98dc43f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Sun, 21 Mar 2021 16:21:05 +0100 Subject: [PATCH] Add full text and similarity search issue10023 review320601007 --- CHANGELOG | 1 + doc/ref/models/fields.rst | 51 +++++- doc/ref/tools/misc.rst | 10 + trytond/backend/database.py | 25 +++ trytond/backend/postgresql/database.py | 171 +++++++++++++++-- trytond/backend/sqlite/database.py | 1 + trytond/ir/lang.py | 3 + trytond/model/fields/__init__.py | 9 +- trytond/model/fields/char.py | 103 ++++++++++- trytond/model/fields/field.py | 34 ++-- trytond/model/fields/text.py | 83 +++++++++ trytond/tests/field_text.py | 7 + trytond/tests/test_field_char.py | 83 +++++---- trytond/tests/test_field_dict.py | 6 +- trytond/tests/test_field_text.py | 242 ++++++++++++++++++++++--- trytond/tests/test_tools.py | 20 +- trytond/tests/test_tryton.py | 36 ++++ trytond/tools/__init__.py | 5 +- trytond/tools/misc.py | 14 ++ 19 files changed, 806 insertions(+), 98 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0df6c3a57d..eed7aec28a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,4 @@ +* Add full text and similarity search and FullText field * Add URL type to icon * Use instance method to get next sequence value * Store the sequence type directly on sequence diff --git a/doc/ref/models/fields.rst b/doc/ref/models/fields.rst index 0f4707a6e2..d44eed7c7e 100644 --- a/doc/ref/models/fields.rst +++ b/doc/ref/models/fields.rst @@ -268,7 +268,15 @@ Char A single line string field. -:class:`Char` has two extra optional arguments: +Search by similarity is used for the ``ilike`` operator and +:meth:`~trytond.tools.is_full_text` value if the backend supports it. +The similarity threshold is defined for the context key ``..search_similarity`` (default value is ``0.3``). + +The field is ordered using the similarity with the context value from the key +``..order`` if it is set. + +:class:`Char` has some extra optional arguments: .. attribute:: Char.size @@ -301,6 +309,26 @@ A single line string field. The database backend must supports unaccented search. +.. attribute:: Char.search_full_text + + If this attribute is set to True, ``ilike`` searches with an + :meth:`~trytond.tools.is_full_text` value use the full text search of the + backend. + The default value is False. + + The context can be used to force the full text search behaviour. + This is done using the key ``..search_full_text``. + If True, the full text search is used no matter what the value. + If False, no full text search is peformed. + + The full text ranking value is added to the similarity if the + ``search_full_text`` is True. + +.. note:: + + The database backend must support full text search otherwise ``ilike`` is + always used. + Text ---- @@ -308,7 +336,7 @@ Text A multi line string field. -:class:`Text` has two extra optional arguments: +:class:`Text` has four extra optional arguments: .. attribute:: Text.size @@ -318,6 +346,25 @@ A multi line string field. Same as :attr:`Char.translate` +.. attribute:: Text.search_unaccented + + Same as :attr:`Char.search_unaccented` + +.. attribute:: Text.search_full_text + + Same as :attr:`Char.search_full_text` + The default value is True. + +FullText +-------- + +.. class:: FullText(\**options) + +An internal field to store a list of parsed strings ordered by weights. +The field is ordered using the full text ranking with the context value from +the key `..order`` if it is set. + + Float ----- diff --git a/doc/ref/tools/misc.rst b/doc/ref/tools/misc.rst index 002cae545c..ad08770b0d 100644 --- a/doc/ref/tools/misc.rst +++ b/doc/ref/tools/misc.rst @@ -9,6 +9,16 @@ Miscellaneous Resolve a dotted name to a global object. +.. method:: unescape_wildcard(string[, wildcards[, escape]]) + +Return the string without the wild card escapes. + +.. method:: is_full_text(value[, escape]) + +Determine if the value can be used as full text search. +This is the case when the value starts and ends with a ``%`` or does not +contain any wild cards. + .. method:: sql_pairing(x, y) Return an SQL expression that pairs SQL integers x and y. diff --git a/trytond/backend/database.py b/trytond/backend/database.py index 0e859f227c..79f93c31f6 100644 --- a/trytond/backend/database.py +++ b/trytond/backend/database.py @@ -181,6 +181,31 @@ class DatabaseInterface(object): "Return the expression to use for unaccentuated columns" return value + def has_similarity(self): + "Return if database supports similarity" + return False + + def similarity(self, column, value): + raise NotImplementedError + + def has_search_full_text(self): + "Return if database supports full text search" + return False + + def format_full_text(self, *documents, language=None): + return '\n'.join(documents) + + def format_full_text_query(self, query, language=None): + raise NotImplementedError + + def search_full_text(self, document, query): + "Return the clause expression for searching document against query" + raise NotImplementedError + + def rank_full_text(self, document, query, normalize=None): + "Return the expression that ranks query on document" + raise NotImplementedError + @classmethod def has_sequence(cls): "Return if database supports sequence querying and assignation" diff --git a/trytond/backend/postgresql/database.py b/trytond/backend/postgresql/database.py index 556bd86ec8..5597326d8f 100644 --- a/trytond/backend/postgresql/database.py +++ b/trytond/backend/postgresql/database.py @@ -8,6 +8,7 @@ import urllib.parse import json from datetime import datetime from decimal import Decimal +from itertools import chain, repeat from threading import RLock try: @@ -31,9 +32,10 @@ from psycopg2 import OperationalError as DatabaseOperationalError from psycopg2 import ProgrammingError from psycopg2.extras import register_default_json, register_default_jsonb -from sql import Flavor, Cast, For +from sql import Flavor, Cast, For, Table +from sql.conditionals import Coalesce from sql.functions import Function -from sql.operators import BinaryOperator +from sql.operators import BinaryOperator, Concat from trytond.backend.database import DatabaseInterface, SQLType from trytond.config import config, parse_uri @@ -80,6 +82,55 @@ class Unaccent(Function): _function = 'unaccent' +class Similarity(Function): + __slots__ = () + _function = 'similarity' + + +class Match(BinaryOperator): + __slots__ = () + _operator = '@@' + + +class ToTsvector(Function): + __slots__ = () + _function = 'to_tsvector' + + +class Setweight(Function): + __slots__ = () + _function = 'setweight' + + +class TsQuery(Function): + __slots__ = () + + +class ToTsQuery(TsQuery): + __slots__ = () + _function = 'to_tsquery' + + +class PlainToTsQuery(TsQuery): + __slots__ = () + _function = 'plainto_tsquery' + + +class PhraseToTsQuery(TsQuery): + __slots__ = () + _function = 'phraseto_tsquery' + + +class WebsearchToTsQuery(TsQuery): + __slots__ = () + _function = 'websearch_to_tsquery' + + +class TsRank(Function): + __slots__ = () + _function = 'ts_rank' + + class AdvisoryLock(Function): _function = 'pg_advisory_xact_lock' @@ -141,7 +192,8 @@ class Database(DatabaseInterface): _current_user = None _has_returning = None _has_select_for_skip_locked = None - _has_unaccent = {} + _has_proc = defaultdict(dict) + _search_full_text_languages = defaultdict(dict) flavor = Flavor(ilike=True) TYPES_MAPPING = { @@ -151,6 +203,7 @@ class Database(DatabaseInterface): 'BLOB': SQLType('BYTEA', 'BYTEA'), 'DATETIME': SQLType('TIMESTAMP', 'TIMESTAMP(0)'), 'TIMESTAMP': SQLType('TIMESTAMP', 'TIMESTAMP(6)'), + 'FULLTEXT': SQLType('TSVECTOR', 'TSVECTOR'), } def __new__(cls, name=_default_name): @@ -468,21 +521,117 @@ class Database(DatabaseInterface): def has_sequence(cls): return True - def has_unaccent(self): - if self.name in self._has_unaccent: - return self._has_unaccent[self.name] + def has_proc(self, name): + if name in self._has_proc[self.name]: + return self._has_proc[self.name][name] connection = self.get_connection() - unaccent = False + result = False try: cursor = connection.cursor() cursor.execute( "SELECT 1 FROM pg_proc WHERE proname=%s", - (Unaccent._function,)) - unaccent = bool(cursor.rowcount) + (name,)) + result = bool(cursor.rowcount) finally: self.put_connection(connection) - self._has_unaccent[self.name] = unaccent - return unaccent + self._has_proc[self.name][name] = result + return result + + def has_unaccent(self): + return self.has_proc(Unaccent._function) + + def has_similarity(self): + return self.has_proc(Similarity._function) + + def similarity(self, column, value): + return Similarity(column, value) + + def has_search_full_text(self): + return True + + def _search_full_text_language(self, language): + languages = self._search_full_text_languages[self.name] + if language not in languages: + lang = Table('ir_lang') + connection = self.get_connection() + try: + cursor = connection.cursor() + cursor.execute(*lang.select( + Coalesce(lang.pg_text_search, 'simple'), + where=lang.code == language, + limit=1)) + config_name, = cursor.fetchone() + finally: + self.put_connection(connection) + languages[language] = config_name + else: + config_name = languages[language] + return config_name + + def format_full_text(self, *documents, language=None): + size = max(len(documents) // 4, 1) + if len(documents) > 1: + weights = chain( + ['A'] * size, ['B'] * size, ['C'] * size, repeat('D')) + else: + weights = [None] + expression = None + if language: + config_name = self._search_full_text_language(language) + else: + config_name = None + for document, weight in zip(documents, weights): + if not document: + continue + if config_name: + ts_vector = ToTsvector(config_name, document) + else: + ts_vector = ToTsvector('simple', document) + if weight: + ts_vector = Setweight(ts_vector, weight) + if expression is None: + expression = ts_vector + else: + expression = Concat(expression, ts_vector) + return expression + + def format_full_text_query(self, query, language=None): + connection = self.get_connection() + try: + version = self.get_version(connection) + finally: + self.put_connection(connection) + if version >= (11, 0): + ToTsQuery = WebsearchToTsQuery + else: + ToTsQuery = PlainToTsQuery + if language: + config_name = self._search_full_text_language(language) + if not isinstance(query, TsQuery): + query = ToTsQuery(config_name, query) + else: + if not isinstance(query, TsQuery): + query = ToTsQuery(query) + return query + + def search_full_text(self, document, query): + return Match(document, query) + + def rank_full_text(self, document, query, normalize=None): + # TODO: weights and cover density + norm_int = 0 + if normalize: + values = { + 'document log': 1, + 'document': 2, + 'mean': 4, + 'word': 8, + 'word log': 16, + 'rank': 32, + } + for norm in normalize: + norm_int |= values.get(norm, 0) + return TsRank(document, query, norm_int) def sql_type(self, type_): if type_ in self.TYPES_MAPPING: diff --git a/trytond/backend/sqlite/database.py b/trytond/backend/sqlite/database.py index 8b8ff5ab74..c378940541 100644 --- a/trytond/backend/sqlite/database.py +++ b/trytond/backend/sqlite/database.py @@ -321,6 +321,7 @@ class Database(DatabaseInterface): 'DATETIME': SQLType('TIMESTAMP', 'TIMESTAMP'), 'BIGINT': SQLType('INTEGER', 'INTEGER'), 'BOOL': SQLType('BOOLEAN', 'BOOLEAN'), + 'FULLTEXT': SQLType('TEXT', 'TEXT'), } def __new__(cls, name=_default_name): diff --git a/trytond/ir/lang.py b/trytond/ir/lang.py index 9e4debdb09..eaeb6c5094 100644 --- a/trytond/ir/lang.py +++ b/trytond/ir/lang.py @@ -73,6 +73,9 @@ class Lang(DeactivableMixin, ModelSQL, ModelView): p_sep_by_space = fields.Boolean('Positive Separate by Space') n_sep_by_space = fields.Boolean('Negative Separate by Space') + pg_text_search = fields.Char( + "PostgreSQL Text Search Configuration", readonly=True) + _lang_cache = Cache('ir.lang') _code_cache = Cache('ir.lang.code', context=False) diff --git a/trytond/model/fields/__init__.py b/trytond/model/fields/__init__.py index ea573e428b..e057735e96 100644 --- a/trytond/model/fields/__init__.py +++ b/trytond/model/fields/__init__.py @@ -7,7 +7,7 @@ from .field import ( from .boolean import Boolean from .integer import Integer, BigInteger from .char import Char -from .text import Text +from .text import Text, FullText from .float import Float from .numeric import Numeric from .date import Date, Timestamp, DateTime, Time, TimeDelta @@ -25,6 +25,7 @@ from .multiselection import MultiSelection __all__ = [ depends, with_inactive_records, SQL_OPERATORS, on_change_result, get_eval_fields, states_validate, domain_validate, context_validate, Field, - Boolean, Integer, BigInteger, Char, Text, Float, Numeric, Date, Timestamp, - DateTime, Time, TimeDelta, Binary, Selection, Reference, Many2One, - One2Many, Many2Many, Function, MultiValue, One2One, Dict, MultiSelection] + Boolean, Integer, BigInteger, Char, Text, FullText, Float, Numeric, Date, + Timestamp, DateTime, Time, TimeDelta, Binary, Selection, Reference, + Many2One, One2Many, Many2Many, Function, MultiValue, One2One, Dict, + MultiSelection] diff --git a/trytond/model/fields/char.py b/trytond/model/fields/char.py index d14ab6b9da..6d5147f07a 100644 --- a/trytond/model/fields/char.py +++ b/trytond/model/fields/char.py @@ -2,7 +2,10 @@ # this repository contains the full copyright notices and license terms. import warnings +from sql.operators import Not + from trytond.rpc import RPC +from trytond.tools import unescape_wildcard, is_full_text from trytond.transaction import Transaction from .field import Field, FieldTranslate, size_validate @@ -14,6 +17,8 @@ class Char(FieldTranslate): _type = 'char' _py_type = str forbidden_chars = '\t\n\r\x0b\x0c' + search_unaccented = True + search_full_text = False def __init__(self, string='', size=None, help='', required=False, readonly=False, domain=None, states=None, translate=False, @@ -37,7 +42,6 @@ class Char(FieldTranslate): self.translate = translate self.__size = None self.size = size - self.search_unaccented = True __init__.__doc__ += Field.__init__.__doc__ def _get_size(self): @@ -75,6 +79,103 @@ class Char(FieldTranslate): value = database.unaccent(value) return value + def convert_domain(self, domain, tables, Model): + transaction = Transaction() + context = transaction.context + database = transaction.database + expression = super().convert_domain(domain, tables, Model) + name, operator, value = domain + if operator.endswith('ilike'): + table, _ = tables[None] + if self.translate: + language = transaction.language + model, join, column = self._get_translation_column( + Model, name) + else: + language = None + column = self.sql_column(table) + column = self._domain_column(operator, column) + + if database.has_similarity() and is_full_text(value): + threshold = context.get( + '%s.%s.search_similarity' % (Model.__name__, name), + context.get('search_similarity', 0.3)) + sim_value = unescape_wildcard(value) + sim_value = self._domain_value(operator, sim_value) + expression = ( + database.similarity(column, sim_value) >= threshold) + if operator.startswith('not'): + expression = Not(expression) + + key = '%s.%s.search_full_text' % (Model.__name__, name) + if ((self.search_full_text or context.get(key)) + and context.get(key, True) + and database.has_search_full_text()): + if context.get(key) or is_full_text(value): + fts_column = database.format_full_text( + column, language=language) + fts_value = value + if key not in context: + fts_value = unescape_wildcard(fts_value) + fts_value = self._domain_value(operator, fts_value) + fts_value = database.format_full_text_query( + fts_value, language=language) + fts = database.search_full_text(fts_column, fts_value) + if operator.startswith('not'): + fts = Not(fts) + if self.translate: + fts = table.id.in_( + join.select(model.id, where=fts)) + if database.has_similarity() and is_full_text(value): + if operator.startswith('not'): + expression |= fts + else: + expression &= fts + else: + expression = fts + return expression + + def convert_order(self, name, tables, Model): + method = getattr(Model, 'order_%s' % name, None) + if method: + return method(tables) + transaction = Transaction() + context = transaction.context + database = transaction.database + key = '%s.%s.order' % (Model.__name__, name) + value = context.get(key) + order = super().convert_order(name, tables, Model) + if value: + expression = None + table, _ = tables[None] + if self.translate: + language = transaction.language + column = self._get_translation_order(tables, Model, name) + else: + language = None + column = self.sql_column(table) + column = self._domain_column('ilike', column) + if database.has_similarity(): + sim_value = unescape_wildcard(value) + sim_value = self._domain_value('ilike', sim_value) + expression = database.similarity(column, sim_value) + key = '%s.%s.search_full_text' % (Model.__name__, name) + if ((self.search_full_text or context.get(key)) + and database.has_search_full_text()): + column = database.format_full_text(column, language=language) + value = self._domain_value('ilike', value) + value = database.format_full_text_query( + value, language=language) + rank = database.rank_full_text( + column, value, normalize=['rank']) + if expression: + expression += rank + else: + expression = rank + if expression: + order = [expression] + return order + def definition(self, model, language): definition = super().definition(model, language) definition['autocomplete'] = list(self.autocomplete) diff --git a/trytond/model/fields/field.py b/trytond/model/fields/field.py index 9fbffac35a..b414d8b777 100644 --- a/trytond/model/fields/field.py +++ b/trytond/model/fields/field.py @@ -544,18 +544,14 @@ class FieldTranslate(Field): return query, from_.join(query, 'LEFT', condition=(query.res_id == res_id)) - def convert_domain(self, domain, tables, Model): + def _get_translation_column(self, Model, name): from trytond.ir.lang import get_parent_language pool = Pool() Translation = pool.get('ir.translation') IrModel = pool.get('ir.model') - if not self.translate: - return super(FieldTranslate, self).convert_domain( - domain, tables, Model) table = join = Model.__table__() model = IrModel.__table__() - name, operator, value = domain language = Transaction().language column = None while language: @@ -564,7 +560,16 @@ class FieldTranslate(Field): Model, name, translation, model, table, join, language) column = Coalesce(NullIf(column, ''), translation.value) language = get_parent_language(language) - column = Coalesce(NullIf(column, ''), self.sql_column(table)) + return table, join, column + + def convert_domain(self, domain, tables, Model): + if not self.translate: + return super(FieldTranslate, self).convert_domain( + domain, tables, Model) + table, _ = tables[None] + name, operator, value = domain + model, join, column = self._get_translation_column(Model, name) + column = Coalesce(NullIf(column, ''), self.sql_column(model)) column = self._domain_column(operator, column) Operator = SQL_OPERATORS[operator] assert name == self.name @@ -574,20 +579,14 @@ class FieldTranslate(Field): elif isinstance(where, operators.NotIn) and not where.right: where = Literal(True) where = self._domain_add_null(column, operator, value, where) - return tables[None][0].id.in_(join.select(table.id, where=where)) + return table.id.in_(join.select(model.id, where=where)) - def convert_order(self, name, tables, Model): + def _get_translation_order(self, tables, Model, name): from trytond.ir.lang import get_parent_language pool = Pool() Translation = pool.get('ir.translation') IrModel = pool.get('ir.model') - if not self.translate: - return super(FieldTranslate, self).convert_order(name, tables, - Model) - assert name == self.name - table, _ = tables[None] - join = table language = Transaction().language column = None @@ -616,7 +615,14 @@ class FieldTranslate(Field): translation, _ = tables[key]['translation'][None] column = Coalesce(NullIf(column, ''), translation.value) language = get_parent_language(language) + return column + def convert_order(self, name, tables, Model): + if not self.translate: + return super().convert_order(name, tables, Model) + assert name == self.name + table, _ = tables[None] + column = self._get_translation_order(tables, Model, name) return [Coalesce(NullIf(column, ''), self.sql_column(table))] def definition(self, model, language): diff --git a/trytond/model/fields/text.py b/trytond/model/fields/text.py index 4b8cb60fa3..2ec1d04af1 100644 --- a/trytond/model/fields/text.py +++ b/trytond/model/fields/text.py @@ -1,5 +1,11 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. +from sql import Query, Expression, Null, Literal +from sql.operators import ILike, Not, In, NotIn + +from trytond.transaction import Transaction + +from .field import Field, SQL_OPERATORS from .char import Char @@ -10,3 +16,80 @@ class Text(Char): _type = 'text' _sql_type = 'TEXT' forbidden_chars = '' + search_full_text = True + + +class FullText(Field): + ''' + Define a full text field. + ''' + _type = 'full_text' + _sql_type = 'FULLTEXT' + _py_type = str + + def sql_format(self, value): + if isinstance(value, (Query, Expression)): + return value + if not value: + return Null + if isinstance(value, (int, float)): + return value + transaction = Transaction() + database = transaction.database + if isinstance(value, str): + value = [value] + return database.format_full_text(*value, language=transaction.language) + + def _rank_column(self, column, name, Model): + transaction = Transaction() + context = transaction.context + database = transaction.database + key = '%s.%s.order' % (Model.__name__, name) + value = context.get(key) + if value and database.has_search_full_text(): + value = database.format_full_text_query( + value, language=transaction.language) + column = database.rank_full_text(column, value, normalize=['rank']) + else: + column = Null + return column + + def convert_domain(self, domain, tables, Model): + transaction = Transaction() + database = transaction.database + table, _ = tables[None] + name, operator, value = domain + assert name == self.name + column = self.sql_column(table) + column = self._domain_column(operator, column) + if operator.endswith('like'): + if database.has_search_full_text(): + value = database.format_full_text_query( + value, language=transaction.language) + expression = database.search_full_text(column, value) + else: + expression = Literal(True) + for v in value.split(): + expression &= ILike(column, '%' + v + '%') + if operator.startswith('not'): + expression = Not(expression) + else: + Operator = SQL_OPERATORS[operator] + column = self._rank_column(column, name, Model) + expression = Operator(column, self._domain_value(operator, value)) + if isinstance(expression, In) and not expression.right: + expression = Literal(False) + elif isinstance(expression, NotIn) and not expression.right: + expression = Literal(True) + expression = self._domain_add_null( + column, operator, value, expression) + return expression + + def convert_order(self, name, tables, Model): + method = getattr(Model, 'order_%s' % name, None) + if method: + return method(tables) + table, _ = tables[None] + column = self.sql_column(table) + column = self._domain_column('ilike', column) + return [self._rank_column(column, name, Model)] diff --git a/trytond/tests/field_text.py b/trytond/tests/field_text.py index d86a9fb8ff..1c4a946cf1 100644 --- a/trytond/tests/field_text.py +++ b/trytond/tests/field_text.py @@ -44,6 +44,12 @@ class TextTranslate(ModelSQL): required=False, translate=True) +class FullText(ModelSQL): + "FullText" + __name__ = 'test.text_full' + full_text = fields.FullText("Full Text") + + def register(module): Pool.register( Text, @@ -51,4 +57,5 @@ def register(module): TextRequired, TextSize, TextTranslate, + FullText, module=module, type_='model') diff --git a/trytond/tests/test_field_char.py b/trytond/tests/test_field_char.py index ce65f2a157..7c92b6b405 100644 --- a/trytond/tests/test_field_char.py +++ b/trytond/tests/test_field_char.py @@ -10,6 +10,7 @@ from trytond.model.exceptions import ( RequiredValidationError, ForbiddenCharValidationError) from trytond.pool import Pool from trytond.tests.test_tryton import activate_module, with_transaction +from trytond.tests.test_tryton import ExtensionTestCase from trytond.transaction import Transaction @@ -494,44 +495,11 @@ class FieldCharTranslatedTestCase(unittest.TestCase, CommonTestCaseMixin): self.assertEqual(char.char, "bar") -class UnaccentedTestCase(unittest.TestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - cls._activate_extension() - - @classmethod - def tearDownClass(cls): - super().tearDownClass() - cls._deactivate_extension() - - @classmethod - @with_transaction() - def _activate_extension(cls): - connection = Transaction().connection - cursor = connection.cursor() - cursor.execute('CREATE EXTENSION "unaccent"') - connection.commit() - cls._clear_unaccent_cache() - - @classmethod - @with_transaction() - def _deactivate_extension(cls): - connection = Transaction().connection - cursor = connection.cursor() - cursor.execute('DROP EXTENSION "unaccent"') - connection.commit() - cls._clear_unaccent_cache() - - @classmethod - def _clear_unaccent_cache(cls): - backend.Database._has_unaccent.clear() - - @unittest.skipUnless(backend.name == 'postgresql', "unaccent works only on postgresql") -class FieldCharUnaccentedTestCase(UnaccentedTestCase): +class FieldCharUnaccentedTestCase(ExtensionTestCase): "Test Field Char with unaccented searches" + extension = 'unaccent' @classmethod def setUpClass(cls): @@ -607,10 +575,55 @@ class FieldCharUnaccentedTestCase(UnaccentedTestCase): self.assertListEqual(chars_ecole, [char]) +@unittest.skipUnless(backend.name == 'postgresql', + "similarity works only on postgresql") +class FieldCharSimilarityTestCase(ExtensionTestCase): + "Test Field Char with similarity searches" + extension = 'pg_trgm' + + @classmethod + def setUpClass(cls): + activate_module('tests') + super().setUpClass() + + @with_transaction() + def test_search(self): + "Test search" + pool = Pool() + Model = pool.get('test.char') + record1, record2 = Model.create([{ + 'char': "word", + }, { + 'char': "Foo", + }]) + + self.assertListEqual(Model.search([ + ('char', 'ilike', 'two words'), + ]), [record1]) + + @with_transaction() + def test_order(self): + "Test order" + pool = Pool() + Model = pool.get('test.char') + record1, record2 = Model.create([{ + 'char': "word", + }, { + 'char': "Foo", + }]) + with Transaction().set_context({ + '%s.char.order' % Model.__name__: 'foo bar', + }): + self.assertListEqual(Model.search([ + ], order=[('char', 'DESC')]), + [record2, record1]) + + def suite(): suite_ = unittest.TestSuite() loader = unittest.TestLoader() suite_.addTests(loader.loadTestsFromTestCase(FieldCharTestCase)) suite_.addTests(loader.loadTestsFromTestCase(FieldCharTranslatedTestCase)) suite_.addTests(loader.loadTestsFromTestCase(FieldCharUnaccentedTestCase)) + suite_.addTests(loader.loadTestsFromTestCase(FieldCharSimilarityTestCase)) return suite_ diff --git a/trytond/tests/test_field_dict.py b/trytond/tests/test_field_dict.py index 37e2b103da..5a98b65fd8 100644 --- a/trytond/tests/test_field_dict.py +++ b/trytond/tests/test_field_dict.py @@ -7,10 +7,9 @@ from trytond.model.dictschema import SelectionError from trytond.model.exceptions import RequiredValidationError from trytond.pool import Pool from trytond.tests.test_tryton import activate_module, with_transaction +from trytond.tests.test_tryton import ExtensionTestCase from trytond.transaction import Transaction -from .test_field_char import UnaccentedTestCase - class FieldDictTestCase(unittest.TestCase): "Test Field Dict" @@ -890,8 +889,9 @@ class FieldDictTestCase(unittest.TestCase): @unittest.skipUnless(backend.name == 'postgresql', "unaccent works only on postgresql") -class FieldDictUnaccentedTestCase(UnaccentedTestCase): +class FieldDictUnaccentedTestCase(ExtensionTestCase): "Test Field Dict with unaccented searched" + extension = 'unaccent' @classmethod def setUpClass(cls): diff --git a/trytond/tests/test_field_text.py b/trytond/tests/test_field_text.py index 6efd5db06a..6ae3bca91a 100644 --- a/trytond/tests/test_field_text.py +++ b/trytond/tests/test_field_text.py @@ -3,10 +3,12 @@ # this repository contains the full copyright notices and license terms. import unittest +from trytond import backend from trytond.model.exceptions import ( RequiredValidationError, SizeValidationError) from trytond.pool import Pool from trytond.tests.test_tryton import activate_module, with_transaction +from trytond.transaction import Transaction class CommonTestCaseMixin: @@ -244,25 +246,73 @@ class CommonTestCaseMixin: text, = Text.create([{ 'text': "Bar", }]) - - texts_bar = Text.search([ - ('text', 'ilike', "bar"), - ]) - texts_b = Text.search([ - ('text', 'ilike', "b%"), - ]) - texts_foo = Text.search([ - ('text', 'ilike', "foo"), - ]) - texts_f = Text.search([ - ('text', 'ilike', "f%"), - ]) + with Transaction().set_context({ + '%s.text.search_full_text' % Text.__name__: False, + }): + texts_bar = Text.search([ + ('text', 'ilike', "bar"), + ]) + texts_b = Text.search([ + ('text', 'ilike', "b%"), + ]) + texts_foo = Text.search([ + ('text', 'ilike', "foo"), + ]) + texts_f = Text.search([ + ('text', 'ilike', "f%"), + ]) self.assertListEqual(texts_bar, [text]) self.assertListEqual(texts_b, [text]) self.assertListEqual(texts_foo, []) self.assertListEqual(texts_f, []) + @unittest.skipIf(backend.name == 'sqlite', + "SQLite does not have full text search") + @with_transaction() + def test_search_full_text(self): + "Test search full text" + Text = self.Text() + fat_cat, = Text.create([{ + 'text': "a fat cat sat on a mat and ate a fat rat", + }]) + sad_cat, = Text.create([{ + 'text': "a sad cat sat", + }]) + + for value, result in [ + ("Fat rat", [fat_cat]), + ('"sad cat" or "fat rat"', [fat_cat, sad_cat]), + ]: + with self.subTest(value=value): + self.assertListEqual(Text.search([ + ('text', 'ilike', value), + ]), result) + + @unittest.skipIf(backend.name == 'sqlite', + "SQLite does not have full text search") + @with_transaction() + def test_search_full_text_order(self): + "Test order search full text" + Text = self.Text() + sad_cat, = Text.create([{ + 'text': "a sad cat sat", + }]) + dog, = Text.create([{ + 'text': "a dog", + }]) + fat_cat, = Text.create([{ + 'text': "a fat cat sat on a mat and ate a fat rat", + }]) + + with Transaction().set_context({ + '%s.text.order' % Text.__name__: ( + '"sad cat" or "fat rat"'), + }): + self.assertListEqual( + Text.search([], order=[('text', 'DESC')]), + [fat_cat, sad_cat, dog]) + @with_transaction() def test_search_not_like(self): "Test search text not like" @@ -297,18 +347,21 @@ class CommonTestCaseMixin: 'text': "Bar", }]) - texts_bar = Text.search([ - ('text', 'not ilike', "bar"), - ]) - texts_b = Text.search([ - ('text', 'not ilike', "b%"), - ]) - texts_foo = Text.search([ - ('text', 'not ilike', "foo"), - ]) - texts_f = Text.search([ - ('text', 'not ilike', "f%"), - ]) + with Transaction().set_context({ + '%s.text.search_full_text' % Text.__name__: False, + }): + texts_bar = Text.search([ + ('text', 'not ilike', "bar"), + ]) + texts_b = Text.search([ + ('text', 'not ilike', "b%"), + ]) + texts_foo = Text.search([ + ('text', 'not ilike', "foo"), + ]) + texts_f = Text.search([ + ('text', 'not ilike', "f%"), + ]) self.assertListEqual(texts_bar, []) self.assertListEqual(texts_b, []) @@ -458,9 +511,148 @@ class FieldTextTranslatedTestCase(unittest.TestCase, CommonTestCaseMixin): return Pool().get('test.text_translate') +class FieldFullTextTestCase(unittest.TestCase): + "Test Field FullText" + + @classmethod + def setUpClass(cls): + activate_module('tests') + + @with_transaction() + def test_create(self): + "Test create full text" + pool = Pool() + Model = pool.get('test.text_full') + + record, = Model.create([{ + 'full_text': "a fat cat sat on a mat and ate a fat rat", + }]) + + self.assertTrue(record.full_text) + + @with_transaction() + def test_create_list(self): + "Test create full text with a list" + pool = Pool() + Model = pool.get('test.text_full') + + record, = Model.create([{ + 'full_text': [ + "The Fat Cats", + "a fat cat sat on a mat and ate a fat rat", + ], + }]) + + self.assertTrue(record.full_text) + + @with_transaction() + def test_search(self): + "Test search full text" + pool = Pool() + Model = pool.get('test.text_full') + + fat_cat, sad_cat = Model.create([{ + 'full_text': "a fat cat sat on a mat and ate a fat rat", + }, { + 'full_text': "a sad cat sat", + }]) + + for clause, result, neg_result in [ + ('fat', [fat_cat], [sad_cat]), + ('cat', [fat_cat, sad_cat], []), + ]: + with self.subTest(clause=clause): + self.assertListEqual(Model.search([ + ('full_text', 'ilike', clause), + ]), result) + self.assertListEqual(Model.search([ + ('full_text', 'not ilike', clause), + ]), neg_result) + + @unittest.skipIf(backend.name == 'sqlite', + "SQLite does not have full text search") + @with_transaction() + def test_search_full_text(self): + "Test search full text" + pool = Pool() + Model = pool.get('test.text_full') + + fat_cat, sad_cat = Model.create([{ + 'full_text': "a fat cat sat on a mat and ate a fat rat", + }, { + 'full_text': "a sad cat sat", + }]) + + for clause, result in [ + ("Fat rat", [fat_cat]), + ('"sad cat" or "fat rat"', [fat_cat, sad_cat]), + ]: + with self.subTest(clause=clause): + self.assertListEqual(Model.search([ + ('full_text', 'ilike', clause), + ]), result) + + @unittest.skipIf(backend.name == 'sqlite', + "SQLite does not have full text search") + @with_transaction() + def test_search_full_text_filter(self): + "Test search full text" + pool = Pool() + Model = pool.get('test.text_full') + + fat_cat, sad_cat = Model.create([{ + 'full_text': "a fat cat sat on a mat and ate a fat rat", + }, { + 'full_text': "a sad cat sat", + }]) + + for clause, result in [ + ("Fat rat", [fat_cat]), + ('"sad cat" or "fat rat"', [fat_cat, sad_cat]), + ]: + with self.subTest(clause=clause): + with Transaction().set_context({ + '%s.full_text.order' % Model.__name__: clause, + }): + self.assertListEqual(Model.search([ + ('full_text', '>', 0.01), + ]), result) + + @unittest.skipIf(backend.name == 'sqlite', + "SQLite does not have full text search") + @with_transaction() + def test_search_full_text_order(self): + "Test order full text" + pool = Pool() + Model = pool.get('test.text_full') + + sad_cat, fat_cat, dog = Model.create([{ + 'full_text': [ + None, + "a fat cat sat on a mat and ate a fat rat", + ], + }, { + 'full_text': [ + "The Fat Cats", + "a fat cat sat on a mat and ate a fat rat", + ] + }, { + 'full_text': "a dog", + }]) + + with Transaction().set_context({ + '%s.full_text.order' % Model.__name__: ( + '"sad cat" or "fat rat"'), + }): + self.assertListEqual( + Model.search([], order=[('full_text', 'DESC')]), + [fat_cat, sad_cat, dog]) + + def suite(): suite_ = unittest.TestSuite() loader = unittest.TestLoader() suite_.addTests(loader.loadTestsFromTestCase(FieldTextTestCase)) suite_.addTests(loader.loadTestsFromTestCase(FieldTextTranslatedTestCase)) + suite_.addTests(loader.loadTestsFromTestCase(FieldFullTextTestCase)) return suite_ diff --git a/trytond/tests/test_tools.py b/trytond/tests/test_tools.py index 8fd1433ad1..ee1be27066 100644 --- a/trytond/tests/test_tools.py +++ b/trytond/tests/test_tools.py @@ -13,7 +13,7 @@ import sql.operators from trytond.tools import ( reduce_ids, reduce_domain, decimal_, is_instance_method, file_open, strip_wildcard, lstrip_wildcard, rstrip_wildcard, slugify, sortable_values, - escape_wildcard, firstline) + escape_wildcard, unescape_wildcard, is_full_text, firstline) from trytond.tools.string_ import StringPartitioned, LazyString from trytond.tools.domain_inversion import ( domain_inversion, parse, simplify, merge, concat, unique_value, @@ -201,6 +201,24 @@ class ToolsTestCase(unittest.TestCase): escape_wildcard('foo%bar_baz\\'), 'foo\\%bar\\_baz\\\\') + def test_unescape_wildcard(self): + "Test unescape_wildcard" + self.assertEqual( + unescape_wildcard('foo\\%bar\\_baz\\\\'), + 'foo%bar_baz\\') + + def test_is_full_text(self): + "Test is_full_text" + for value, result in [ + ('foo', True), + ('%foo bar%', True), + ('foo%', False), + ('foo_bar', False), + ('foo\\_bar', True), + ]: + with self.subTest(value=value): + self.assertEqual(is_full_text(value), result) + def test_slugify(self): "Test slugify" self.assertEqual(slugify('unicode ♥ is ☢'), 'unicode-is') diff --git a/trytond/tests/test_tryton.py b/trytond/tests/test_tryton.py index 5adb0dd48c..28953b9670 100644 --- a/trytond/tests/test_tryton.py +++ b/trytond/tests/test_tryton.py @@ -842,6 +842,42 @@ def create_db(name=DB_NAME, lang='en'): Module.update_list() +class ExtensionTestCase(unittest.TestCase): + extension = None + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._activate_extension() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + cls._deactivate_extension() + + @classmethod + @with_transaction() + def _activate_extension(cls): + connection = Transaction().connection + cursor = connection.cursor() + cursor.execute('CREATE EXTENSION "%s"' % cls.extension) + connection.commit() + cls._clear_cache() + + @classmethod + @with_transaction() + def _deactivate_extension(cls): + connection = Transaction().connection + cursor = connection.cursor() + cursor.execute('DROP EXTENSION "%s"' % cls.extension) + connection.commit() + cls._clear_cache() + + @classmethod + def _clear_cache(cls): + backend.Database._has_proc.clear() + + def drop_db(name=DB_NAME): if db_exist(name): database = backend.Database(name) diff --git a/trytond/tools/__init__.py b/trytond/tools/__init__.py index e1f769fae0..99ae849d35 100644 --- a/trytond/tools/__init__.py +++ b/trytond/tools/__init__.py @@ -9,14 +9,15 @@ from .misc import ( file_open, get_smtp_server, reduce_ids, reduce_domain, grouped_slice, is_instance_method, resolve, strip_wildcard, lstrip_wildcard, rstrip_wildcard, slugify, sortable_values, - escape_wildcard, sql_pairing, firstline) + escape_wildcard, unescape_wildcard, is_full_text, sql_pairing, firstline) from .decimal_ import decistmt __all__ = ['file_open', 'get_smtp_server', 'reduce_ids', 'reduce_domain', 'grouped_slice', 'is_instance_method', 'resolve', 'strip_wildcard', 'lstrip_wildcard', 'rstrip_wildcard', 'slugify', 'decistmt', 'ClassProperty', 'cursor_dict', 'cached_property', - 'sortable_values', 'escape_wildcard', 'sql_pairing', 'firstline'] + 'sortable_values', 'escape_wildcard', 'unescape_wildcard', 'is_full_text', + 'sql_pairing', 'firstline'] class ClassProperty(property): diff --git a/trytond/tools/misc.py b/trytond/tools/misc.py index 5377c9ed38..7bdf19430b 100644 --- a/trytond/tools/misc.py +++ b/trytond/tools/misc.py @@ -221,6 +221,20 @@ def escape_wildcard(string, wildcards='%_', escape='\\'): return string +def unescape_wildcard(string, wildcards='%_', escape='\\'): + for wildcard in wildcards + escape: + string = string.replace(escape + wildcard, wildcard) + return string + + +def is_full_text(value, escape='\\'): + escaped = value.strip('%') + escaped = escaped.replace(escape + '%', '').replace(escape + '_', '') + if '%' in escaped or '_' in escaped: + return False + return value.startswith('%') == value.endswith('%') + + _slugify_strip_re = re.compile(r'[^\w\s-]') _slugify_hyphenate_re = re.compile(r'[-\s]+')