Guess encoding when sending bytes to a Char widget

issue8824
review258401002

tryton/common/htmltextbuffer.py

@@ -9,6 +9,17 @@ from html.parser import HTMLParser
 
 from gi.repository import Gtk, Gdk, Pango
 
+
+def guess_decode(bytes, errors='strict'):
+    for encoding in [sys.getfilesystemencoding(), 'utf-8', 'utf-16', 'utf-32']:
+        try:
+            return bytes.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    else:
+        return bytes.decode('ascii', errors=errors)
+
+
 MIME = Gdk.Atom.intern('text/html', False)
 # Disable serialize/deserialize registration function because it does not work
 # on GTK-3, the "guint8 *data" is converted into a Gtk.TextIter
@@ -269,15 +280,7 @@ def serialize(register, content, start, end, data):
 
 def deserialize(register, content, iter_, text, create_tags, data):
     if not isinstance(text, str):
-        for encoding in [sys.getfilesystemencoding(),
-                'utf-8', 'utf-16', 'utf-32']:
-            try:
-                text = text.decode(encoding)
-            except UnicodeDecodeError:
-                continue
-            break
-        else:
-            text = text.decode('ascii', errors='replace')
+        text = guess_decode(text, errors='replace')
     text, tags = parse_markup(normalize_markup(text, method='xml'))
     offset = iter_.get_offset()
     content.insert(iter_, text)

tryton/gui/window/view_form/model/field.py

@@ -3,6 +3,7 @@
 import os
 from itertools import chain
 import tempfile
+import logging
 import locale
 from tryton.common import \
         domain_inversion, eval_domain, localize_domain, \
@@ -15,9 +16,12 @@ import decimal
 from decimal import Decimal
 import math
 from tryton.common import RPCExecute, RPCException
+from tryton.common.htmltextbuffer import guess_decode
 from tryton.pyson import PYSONDecoder
 from tryton.config import CONFIG
 
+logger = logging.getLogger(__name__)
+
 
 class Field(object):
     '''
@@ -203,6 +207,17 @@ class CharField(Field):
     def get(self, record):
         return super(CharField, self).get(record) or self._default
 
+    def set_client(self, record, value, force_change=False):
+        if isinstance(value, bytes):
+            try:
+                value = guess_decode(value)
+            except UnicodeDecodeError:
+                logger.warning(
+                    "The encoding can not be guessed for field '%(name)s'",
+                    {'name': self.name})
+                value = None
+        super().set_client(record, value, force_change)
+
 
 class SelectionField(Field):