From acdbe5a08fbdeda39bc84c33459728e167373f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20=C3=89vrard?= Date: Fri, 16 Mar 2018 18:17:53 +0100 Subject: [PATCH] Allow to perform unaccented searches on Char fields issue7115 review44611002 --- CHANGELOG | 1 + doc/ref/models/fields.rst | 9 ++ trytond/backend/database.py | 8 ++ trytond/backend/postgresql/database.py | 28 +++++++ trytond/model/fields/char.py | 16 ++++ trytond/model/fields/field.py | 5 ++ trytond/tests/field_char.py | 26 ++++++ trytond/tests/test_field_char.py | 111 +++++++++++++++++++++++++ 8 files changed, 204 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index fb047924e3..a22cabfe03 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,4 @@ +* Allow to perform unaccented searches on Char fields * Always raise exception in table handler * Simplify API for Session: new, remove, check and reset * Ensure that all buttons are registered in ir.model.button diff --git a/doc/ref/models/fields.rst b/doc/ref/models/fields.rst index cb6bfdaa32..b25f5d0740 100644 --- a/doc/ref/models/fields.rst +++ b/doc/ref/models/fields.rst @@ -280,6 +280,15 @@ A single line string field. ComboboxEntry in the client. The set of field names could be filled by using the decorator :meth:`depends`. +.. attribute:: Char.search_unaccented + + If this attribute is set to True, ``ilike`` searches will be performed on + unaccented strings. The default value is True. + +.. warning:: + + The database backend must supports unaccented search. + Text ---- diff --git a/trytond/backend/database.py b/trytond/backend/database.py index 69a794f27a..350b95c092 100644 --- a/trytond/backend/database.py +++ b/trytond/backend/database.py @@ -164,6 +164,14 @@ class DatabaseInterface(object): "Return if database supports window functions." return False + def has_unaccent(self): + "Return if database supports unaccentuated searches" + return False + + def unaccent(self, value): + "Return the expression to use for unaccentuated columns" + return value + @classmethod def has_sequence(cls): "Return if database supports sequence querying and assignation" diff --git a/trytond/backend/postgresql/database.py b/trytond/backend/postgresql/database.py index bbe0bc1fb2..620a3f8c45 100644 --- a/trytond/backend/postgresql/database.py +++ b/trytond/backend/postgresql/database.py @@ -29,6 +29,7 @@ from psycopg2 import OperationalError as DatabaseOperationalError from psycopg2.extras import register_default_json, register_default_jsonb from sql import Flavor +from sql.functions import Function from trytond.backend.database import DatabaseInterface, SQLType from trytond.config import config, parse_uri @@ -60,6 +61,11 @@ class LoggingCursor(cursor): cursor.execute(self, sql, args) +class Unaccent(Function): + __slots__ = () + _function = 'unaccent' + + class Database(DatabaseInterface): _lock = RLock() @@ -71,6 +77,7 @@ class Database(DatabaseInterface): _search_path = None _current_user = None _has_returning = None + _has_unaccent = {} flavor = Flavor(ilike=True) TYPES_MAPPING = { @@ -325,6 +332,22 @@ class Database(DatabaseInterface): def has_sequence(cls): return True + def has_unaccent(self): + if self.name in self._has_unaccent: + return self._has_unaccent[self.name] + connection = self.get_connection() + unaccent = False + try: + cursor = connection.cursor() + cursor.execute( + "SELECT 1 FROM pg_proc WHERE proname=%s", + (Unaccent._function,)) + unaccent = bool(cursor.rowcount) + finally: + self.put_connection(connection) + self._has_unaccent[self.name] = unaccent + return unaccent + def sql_type(self, type_): if type_ in self.TYPES_MAPPING: return self.TYPES_MAPPING[type_] @@ -338,6 +361,11 @@ class Database(DatabaseInterface): return Binary(value) return value + def unaccent(self, value): + if self.has_unaccent(): + return Unaccent(value) + return value + def sequence_exist(self, connection, name): cursor = connection.cursor() for schema in self.search_path: diff --git a/trytond/model/fields/char.py b/trytond/model/fields/char.py index 1ea74ac7d9..dfef87711d 100644 --- a/trytond/model/fields/char.py +++ b/trytond/model/fields/char.py @@ -3,6 +3,7 @@ import sys import warnings +from trytond.transaction import Transaction from .field import Field, FieldTranslate, size_validate from ...rpc import RPC @@ -35,6 +36,7 @@ class Char(FieldTranslate): self.translate = translate self.__size = None self.size = size + self.search_unaccented = True __init__.__doc__ += Field.__init__.__doc__ def _get_size(self): @@ -65,3 +67,17 @@ class Char(FieldTranslate): assert hasattr(model, func_name), \ 'Missing %s on model %s' % (func_name, model.__name__) model.__rpc__.setdefault(func_name, RPC(instantiate=0)) + + def _domain_column(self, operator, column): + column = super(Char, self)._domain_column(operator, column) + if self.search_unaccented and operator.endswith('ilike'): + database = Transaction().database + column = database.unaccent(column) + return column + + def _domain_value(self, operator, value): + value = super(Char, self)._domain_value(operator, value) + if self.search_unaccented and operator.endswith('ilike'): + database = Transaction().database + value = database.unaccent(value) + return value diff --git a/trytond/model/fields/field.py b/trytond/model/fields/field.py index 2f3c9ed70b..7af334d08f 100644 --- a/trytond/model/fields/field.py +++ b/trytond/model/fields/field.py @@ -302,6 +302,9 @@ class Field(object): def sql_column(self, table): return Column(table, self.name) + def _domain_column(self, operator, column): + return column + def _domain_value(self, operator, value): if isinstance(value, (Select, CombiningQuery)): return value @@ -330,6 +333,7 @@ class Field(object): return method(domain, tables) Operator = SQL_OPERATORS[operator] column = self.sql_column(table) + column = self._domain_column(operator, column) expression = Operator(column, self._domain_value(operator, value)) if isinstance(expression, operators.In) and not expression.right: expression = Literal(False) @@ -417,6 +421,7 @@ class FieldTranslate(Field): column = Coalesce(NullIf(column, ''), translation.value) language = get_parent_language(language) column = Coalesce(NullIf(column, ''), self.sql_column(table)) + column = self._domain_column(operator, column) Operator = SQL_OPERATORS[operator] assert name == self.name where = Operator(column, self._domain_value(operator, value)) diff --git a/trytond/tests/field_char.py b/trytond/tests/field_char.py index 8947f8ac39..1214027759 100644 --- a/trytond/tests/field_char.py +++ b/trytond/tests/field_char.py @@ -39,6 +39,29 @@ class CharTranslate(ModelSQL): char = fields.Char("Char", translate=True) +class CharUnaccentedOn(ModelSQL): + 'Char Unaccented' + __name__ = 'test.char_unaccented_on' + char = fields.Char("Char") + + +class CharUnaccentedOff(ModelSQL): + 'Char Unaccented' + __name__ = 'test.char_unaccented_off' + char = fields.Char("Char") + + @classmethod + def __setup__(cls): + super(CharUnaccentedOff, cls).__setup__() + cls.char.search_unaccented = False + + +class CharUnaccentedTranslate(ModelSQL): + 'Char Unaccented Translate' + __name__ = 'test.char_unaccented_translate' + char = fields.Char("Char", translate=True) + + def register(module): Pool.register( Char, @@ -46,4 +69,7 @@ def register(module): CharRequired, CharSize, CharTranslate, + CharUnaccentedOn, + CharUnaccentedOff, + CharUnaccentedTranslate, module=module, type_='model') diff --git a/trytond/tests/test_field_char.py b/trytond/tests/test_field_char.py index e7ebab6194..783e03bca8 100644 --- a/trytond/tests/test_field_char.py +++ b/trytond/tests/test_field_char.py @@ -3,9 +3,11 @@ # this repository contains the full copyright notices and license terms. import unittest +from trytond import backend from trytond.exceptions import UserError from trytond.pool import Pool from trytond.tests.test_tryton import activate_module, with_transaction +from trytond.transaction import Transaction class CommonTestCaseMixin: @@ -452,9 +454,118 @@ class FieldCharTranslatedTestCase(unittest.TestCase, CommonTestCaseMixin): return Pool().get('test.char_translate') +@unittest.skipUnless(backend.name() == 'postgresql', + "unaccent works only on postgresql") +class FieldCharUnaccentedTestCase(unittest.TestCase): + "Test Field Char with unaccented searches" + + @classmethod + def setUpClass(cls): + super(FieldCharUnaccentedTestCase, cls).setUpClass() + activate_module('tests') + cls._activate_extension() + + @classmethod + def tearDownClass(cls): + super(FieldCharUnaccentedTestCase, cls).tearDownClass() + cls._deactivate_extension() + + @classmethod + @with_transaction() + def _activate_extension(cls): + connection = Transaction().connection + cursor = connection.cursor() + cursor.execute('CREATE EXTENSION "unaccent"') + connection.commit() + cls._clear_unaccent_cache() + + @classmethod + @with_transaction() + def _deactivate_extension(cls): + connection = Transaction().connection + cursor = connection.cursor() + cursor.execute('DROP EXTENSION "unaccent"') + connection.commit() + cls._clear_unaccent_cache() + + @classmethod + def _clear_unaccent_cache(cls): + Database = backend.get('Database') + Database._has_unaccent.clear() + + @with_transaction() + def test_normal_search(self): + "Test searches without the unaccented feature" + Char = Pool().get('test.char_unaccented_off') + char, = Char.create([{ + 'char': u'Stéphanie', + }]) + + chars_stephanie = Char.search([ + ('char', 'ilike', 'Stephanie'), + ]) + + self.assertListEqual(chars_stephanie, []) + + @with_transaction() + def test_accented_search(self): + "Test searches of accented value" + Char = Pool().get('test.char_unaccented_on') + char, = Char.create([{ + 'char': u'Stéphanie', + }]) + + chars_stephanie = Char.search([ + ('char', 'ilike', 'Stephanie'), + ]) + + self.assertListEqual(chars_stephanie, [char]) + + @with_transaction() + def test_unaccented_search(self): + "Test searches of unaccented value" + Char = Pool().get('test.char_unaccented_on') + char, = Char.create([{ + 'char': u'Stephanie', + }]) + + chars_stephanie = Char.search([ + ('char', 'ilike', 'Stéphanie'), + ]) + self.assertListEqual(chars_stephanie, [char]) + + @with_transaction() + def test_unaccented_translated_search(self): + "Test unaccented translated search" + pool = Pool() + Char = pool.get('test.char_unaccented_translate') + Lang = pool.get('ir.lang') + + lang, = Lang.search([ + ('translatable', '=', False), + ('code', '!=', 'en'), + ], limit=1) + lang.translatable = True + lang.save() + char, = Char.create([{ + 'char': u'School', + }]) + + with Transaction().set_context(lang=lang.code): + trans_char = Char(char.id) + trans_char.char = u'École' + trans_char.save() + + chars_ecole = Char.search([ + ('char', 'ilike', 'Ecole'), + ]) + self.assertListEqual(chars_ecole, [char]) + + def suite(): suite_ = unittest.TestSuite() loader = unittest.TestLoader() suite_.addTests(loader.loadTestsFromTestCase(FieldCharTestCase)) suite_.addTests(loader.loadTestsFromTestCase(FieldCharTranslatedTestCase)) + suite_.addTests(loader.loadTestsFromTestCase(FieldCharUnaccentedTestCase)) return suite_