Allow to perform unaccented searches on Char fields

issue7115
review44611002

CHANGELOG

@@ -1,3 +1,4 @@
+* Allow to perform unaccented searches on Char fields
 * Always raise exception in table handler
 * Simplify API for Session: new, remove, check and reset
 * Ensure that all buttons are registered in ir.model.button

doc/ref/models/fields.rst

@@ -280,6 +280,15 @@ A single line string field.
     ComboboxEntry in the client.
     The set of field names could be filled by using the decorator :meth:`depends`.
 
+.. attribute:: Char.search_unaccented
+
+    If this attribute is set to True, ``ilike`` searches will be performed on
+    unaccented strings. The default value is True.
+
+.. warning::
+
+    The database backend must supports unaccented search.
+
 Text
 ----
 

trytond/backend/database.py

@@ -164,6 +164,14 @@ class DatabaseInterface(object):
         "Return if database supports window functions."
         return False
 
+    def has_unaccent(self):
+        "Return if database supports unaccentuated searches"
+        return False
+
+    def unaccent(self, value):
+        "Return the expression to use for unaccentuated columns"
+        return value
+
     @classmethod
     def has_sequence(cls):
         "Return if database supports sequence querying and assignation"

trytond/backend/postgresql/database.py

@@ -29,6 +29,7 @@ from psycopg2 import OperationalError as DatabaseOperationalError
 from psycopg2.extras import register_default_json, register_default_jsonb
 
 from sql import Flavor
+from sql.functions import Function
 
 from trytond.backend.database import DatabaseInterface, SQLType
 from trytond.config import config, parse_uri
@@ -60,6 +61,11 @@ class LoggingCursor(cursor):
         cursor.execute(self, sql, args)
 
 
+class Unaccent(Function):
+    __slots__ = ()
+    _function = 'unaccent'
+
+
 class Database(DatabaseInterface):
 
     _lock = RLock()
@@ -71,6 +77,7 @@ class Database(DatabaseInterface):
     _search_path = None
     _current_user = None
     _has_returning = None
+    _has_unaccent = {}
     flavor = Flavor(ilike=True)
 
     TYPES_MAPPING = {
@@ -325,6 +332,22 @@ class Database(DatabaseInterface):
     def has_sequence(cls):
         return True
 
+    def has_unaccent(self):
+        if self.name in self._has_unaccent:
+            return self._has_unaccent[self.name]
+        connection = self.get_connection()
+        unaccent = False
+        try:
+            cursor = connection.cursor()
+            cursor.execute(
+                "SELECT 1 FROM pg_proc WHERE proname=%s",
+                (Unaccent._function,))
+            unaccent = bool(cursor.rowcount)
+        finally:
+            self.put_connection(connection)
+        self._has_unaccent[self.name] = unaccent
+        return unaccent
+
     def sql_type(self, type_):
         if type_ in self.TYPES_MAPPING:
             return self.TYPES_MAPPING[type_]
@@ -338,6 +361,11 @@ class Database(DatabaseInterface):
                 return Binary(value)
         return value
 
+    def unaccent(self, value):
+        if self.has_unaccent():
+            return Unaccent(value)
+        return value
+
     def sequence_exist(self, connection, name):
         cursor = connection.cursor()
         for schema in self.search_path:

trytond/model/fields/char.py

@@ -3,6 +3,7 @@
 import sys
 import warnings
 
+from trytond.transaction import Transaction
 from .field import Field, FieldTranslate, size_validate
 from ...rpc import RPC
 
@@ -35,6 +36,7 @@ class Char(FieldTranslate):
         self.translate = translate
         self.__size = None
         self.size = size
+        self.search_unaccented = True
     __init__.__doc__ += Field.__init__.__doc__
 
     def _get_size(self):
@@ -65,3 +67,17 @@ class Char(FieldTranslate):
             assert hasattr(model, func_name), \
                 'Missing %s on model %s' % (func_name, model.__name__)
             model.__rpc__.setdefault(func_name, RPC(instantiate=0))
+
+    def _domain_column(self, operator, column):
+        column = super(Char, self)._domain_column(operator, column)
+        if self.search_unaccented and operator.endswith('ilike'):
+            database = Transaction().database
+            column = database.unaccent(column)
+        return column
+
+    def _domain_value(self, operator, value):
+        value = super(Char, self)._domain_value(operator, value)
+        if self.search_unaccented and operator.endswith('ilike'):
+            database = Transaction().database
+            value = database.unaccent(value)
+        return value

trytond/model/fields/field.py

@@ -302,6 +302,9 @@ class Field(object):
     def sql_column(self, table):
         return Column(table, self.name)
 
+    def _domain_column(self, operator, column):
+        return column
+
     def _domain_value(self, operator, value):
         if isinstance(value, (Select, CombiningQuery)):
             return value
@@ -330,6 +333,7 @@ class Field(object):
             return method(domain, tables)
         Operator = SQL_OPERATORS[operator]
         column = self.sql_column(table)
+        column = self._domain_column(operator, column)
         expression = Operator(column, self._domain_value(operator, value))
         if isinstance(expression, operators.In) and not expression.right:
             expression = Literal(False)
@@ -417,6 +421,7 @@ class FieldTranslate(Field):
             column = Coalesce(NullIf(column, ''), translation.value)
             language = get_parent_language(language)
         column = Coalesce(NullIf(column, ''), self.sql_column(table))
+        column = self._domain_column(operator, column)
         Operator = SQL_OPERATORS[operator]
         assert name == self.name
         where = Operator(column, self._domain_value(operator, value))

trytond/tests/field_char.py

@@ -39,6 +39,29 @@ class CharTranslate(ModelSQL):
     char = fields.Char("Char", translate=True)
 
 
+class CharUnaccentedOn(ModelSQL):
+    'Char Unaccented'
+    __name__ = 'test.char_unaccented_on'
+    char = fields.Char("Char")
+
+
+class CharUnaccentedOff(ModelSQL):
+    'Char Unaccented'
+    __name__ = 'test.char_unaccented_off'
+    char = fields.Char("Char")
+
+    @classmethod
+    def __setup__(cls):
+        super(CharUnaccentedOff, cls).__setup__()
+        cls.char.search_unaccented = False
+
+
+class CharUnaccentedTranslate(ModelSQL):
+    'Char Unaccented Translate'
+    __name__ = 'test.char_unaccented_translate'
+    char = fields.Char("Char", translate=True)
+
+
 def register(module):
     Pool.register(
         Char,
@@ -46,4 +69,7 @@ def register(module):
         CharRequired,
         CharSize,
         CharTranslate,
+        CharUnaccentedOn,
+        CharUnaccentedOff,
+        CharUnaccentedTranslate,
         module=module, type_='model')

trytond/tests/test_field_char.py

@@ -3,9 +3,11 @@
 # this repository contains the full copyright notices and license terms.
 import unittest
 
+from trytond import backend
 from trytond.exceptions import UserError
 from trytond.pool import Pool
 from trytond.tests.test_tryton import activate_module, with_transaction
+from trytond.transaction import Transaction
 
 
 class CommonTestCaseMixin:
@@ -452,9 +454,118 @@ class FieldCharTranslatedTestCase(unittest.TestCase, CommonTestCaseMixin):
         return Pool().get('test.char_translate')
 
 
+@unittest.skipUnless(backend.name() == 'postgresql',
+    "unaccent works only on postgresql")
+class FieldCharUnaccentedTestCase(unittest.TestCase):
+    "Test Field Char with unaccented searches"
+
+    @classmethod
+    def setUpClass(cls):
+        super(FieldCharUnaccentedTestCase, cls).setUpClass()
+        activate_module('tests')
+        cls._activate_extension()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(FieldCharUnaccentedTestCase, cls).tearDownClass()
+        cls._deactivate_extension()
+
+    @classmethod
+    @with_transaction()
+    def _activate_extension(cls):
+        connection = Transaction().connection
+        cursor = connection.cursor()
+        cursor.execute('CREATE EXTENSION "unaccent"')
+        connection.commit()
+        cls._clear_unaccent_cache()
+
+    @classmethod
+    @with_transaction()
+    def _deactivate_extension(cls):
+        connection = Transaction().connection
+        cursor = connection.cursor()
+        cursor.execute('DROP EXTENSION "unaccent"')
+        connection.commit()
+        cls._clear_unaccent_cache()
+
+    @classmethod
+    def _clear_unaccent_cache(cls):
+        Database = backend.get('Database')
+        Database._has_unaccent.clear()
+
+    @with_transaction()
+    def test_normal_search(self):
+        "Test searches without the unaccented feature"
+        Char = Pool().get('test.char_unaccented_off')
+        char, = Char.create([{
+                    'char': u'Stéphanie',
+                    }])
+
+        chars_stephanie = Char.search([
+                ('char', 'ilike', 'Stephanie'),
+                ])
+
+        self.assertListEqual(chars_stephanie, [])
+
+    @with_transaction()
+    def test_accented_search(self):
+        "Test searches of accented value"
+        Char = Pool().get('test.char_unaccented_on')
+        char, = Char.create([{
+                    'char': u'Stéphanie',
+                    }])
+
+        chars_stephanie = Char.search([
+                ('char', 'ilike', 'Stephanie'),
+                ])
+
+        self.assertListEqual(chars_stephanie, [char])
+
+    @with_transaction()
+    def test_unaccented_search(self):
+        "Test searches of unaccented value"
+        Char = Pool().get('test.char_unaccented_on')
+        char, = Char.create([{
+                    'char': u'Stephanie',
+                    }])
+
+        chars_stephanie = Char.search([
+                ('char', 'ilike', 'Stéphanie'),
+                ])
+        self.assertListEqual(chars_stephanie, [char])
+
+    @with_transaction()
+    def test_unaccented_translated_search(self):
+        "Test unaccented translated search"
+        pool = Pool()
+        Char = pool.get('test.char_unaccented_translate')
+        Lang = pool.get('ir.lang')
+
+        lang, = Lang.search([
+                ('translatable', '=', False),
+                ('code', '!=', 'en'),
+                ], limit=1)
+        lang.translatable = True
+        lang.save()
+        char, = Char.create([{
+                    'char': u'School',
+                    }])
+
+        with Transaction().set_context(lang=lang.code):
+            trans_char = Char(char.id)
+            trans_char.char = u'École'
+            trans_char.save()
+
+            chars_ecole = Char.search([
+                    ('char', 'ilike', 'Ecole'),
+                    ])
+            self.assertListEqual(chars_ecole, [char])
+
+
 def suite():
     suite_ = unittest.TestSuite()
     loader = unittest.TestLoader()
     suite_.addTests(loader.loadTestsFromTestCase(FieldCharTestCase))
     suite_.addTests(loader.loadTestsFromTestCase(FieldCharTranslatedTestCase))
+    suite_.addTests(loader.loadTestsFromTestCase(FieldCharUnaccentedTestCase))
     return suite_