Add decimal digits validation on fields and fix Float and Numeric for lost of precision

CHANGELOG

@@ -1,3 +1,5 @@
+* Fix Float and Numeric for lost of precision
+* Add decimal digits validation on fields
 * Improve unittests to run trytond from tests
   Use sqlite as default backend
   Add skiptest attribute to data tag

README

@@ -47,7 +47,8 @@ available. Here is some warnings about using other backends:
 
     * MySQL can not create indexes containing text or blob fields.
 
-    * Tryton uses a DECIMAL(16, 8) for Decimal fields.
+    * Tryton uses a DECIMAL(65, 30) for Decimal fields and DOUBLE(255, 30) for
+      Float fields.
 
   * SQLite
 

trytond/backend/fields.py

@@ -1,7 +1,7 @@
 #This file is part of Tryton.  The COPYRIGHT file at the top level of
 #this repository contains the full copyright notices and license terms.
 import datetime
-
+from decimal import Decimal
 try:
     import hashlib
 except ImportError:
@@ -74,7 +74,15 @@ class Float(Field):
 
 
 class Numeric(Float):
-    pass
+
+    @staticmethod
+    def sql_format(value):
+        if not value:
+            value = Decimal('0.0')
+        if isinstance(value, (int, long, float)):
+            value = Decimal(repr(value))
+        assert isinstance(value, Decimal)
+        return value
 
 
 class Date(Field):

trytond/backend/mysql/database.py

@@ -5,6 +5,7 @@ from trytond.backend.database import DatabaseInterface, CursorInterface
 from trytond.config import CONFIG
 import MySQLdb
 import MySQLdb.cursors
+import MySQLdb.converters
 from MySQLdb import IntegrityError as DatabaseIntegrityError
 from MySQLdb import OperationalError as DatabaseOperationalError
 import logging
@@ -27,11 +28,14 @@ class Database(DatabaseInterface):
         return self
 
     def cursor(self, autocommit=False):
+        conv = MySQLdb.converters.conversions.copy()
+        conv[float] = lambda value, _: repr(value)
         args = {
             'db': self.database_name,
             'sql_mode': 'traditional,postgresql',
             'use_unicode': True,
             'charset': 'utf8',
+            'conv': conv,
         }
         if CONFIG['db_host']:
             args['host'] = CONFIG['db_host']
@@ -50,9 +54,11 @@ class Database(DatabaseInterface):
     def create(self, cursor, database_name):
         cursor.execute('CREATE DATABASE `' + database_name + '` ' \
                 'DEFAULT CHARACTER SET = \'utf8\'')
+        Database._list_cache = None
 
     def drop(self, cursor, database_name):
         cursor.execute('DROP DATABASE `' + database_name + '`')
+        Database._list_cache = None
 
     @staticmethod
     def dump(database_name):

trytond/backend/mysql/fields.py

@@ -55,14 +55,14 @@ class Float(fields.Float):
 
     @staticmethod
     def sql_type(field):
-        return ('double', 'double')
+        return ('double', 'double(255, 30)')
 
 
 class Numeric(fields.Numeric):
 
     @staticmethod
     def sql_type(field):
-        return ('decimal', 'decimal(16, 8)')
+        return ('decimal', 'decimal(65, 30)')
 
 
 class Date(fields.Date):

trytond/backend/mysql/table.py

@@ -397,7 +397,9 @@ class TableHandler(TableHandlerInterface):
                 size = 255
             res = 'varchar(%s)' % str(size)
         elif typname == 'decimal':
-            res = 'decimal(16, 8)'
+            res = 'decimal(65, 30)'
+        elif typname == 'double':
+            res = 'double(255, 30)'
         else:
             res = typname
         # Default value for timestamp doesn't work

trytond/backend/postgresql/database.py

@@ -18,6 +18,7 @@ import re
 import os
 if os.name == 'posix':
     import pwd
+from decimal import Decimal
 
 RE_FROM = re.compile('.* from "?([a-zA-Z_0-9]+)"?.*$')
 RE_INTO = re.compile('.* into "?([a-zA-Z_0-9]+)"?.*$')
@@ -395,3 +396,5 @@ class Cursor(CursorInterface):
 
 register_type(UNICODE)
 register_adapter(Session, AsIs)
+register_adapter(float, lambda value: AsIs(repr(value)))
+register_adapter(Decimal, lambda value: AsIs(str(value)))

trytond/backend/sqlite/database.py

@@ -353,8 +353,8 @@ class Cursor(CursorInterface):
             select += ' OFFSET %d' % offset
         return select
 
-sqlite.register_converter('NUMERIC', lambda val: Decimal(str(val)))
-sqlite.register_adapter(Decimal, float)
+sqlite.register_converter('NUMERIC', lambda val: Decimal(val))
+sqlite.register_adapter(Decimal, lambda val: buffer(str(val)))
 sqlite.register_adapter(Session, int)
 def adapt_datetime(val):
     return val.replace(tzinfo=None).isoformat(" ")

trytond/ir/translation.py

@@ -900,6 +900,7 @@ class TranslationClean(Wizard):
                             'domain_validation_record',
                             'required_validation_record',
                             'size_validation_record',
+                            'digits_validation_record',
                             'access_error',
                             'read_error',
                             'write_error',

trytond/ir/translation.xml

@@ -178,6 +178,15 @@ this repository contains the full copyright notices and license terms. -->
             <field name="module">ir</field>
             <field name="fuzzy" eval="False"/>
         </record>
+        <record model="ir.translation" id="translation_digits_validation_record">
+            <field name="name">digits_validation_record</field>
+            <field name="lang">en_US</field>
+            <field name="type">error</field>
+            <field name="src">The field "%s" on "%s" has too many decimal digits.</field>
+            <field name="value">The field "%s" on "%s" has too many decimal digits.</field>
+            <field name="module">ir</field>
+            <field name="fuzzy" eval="False"/>
+        </record>
         <record model="ir.translation" id="translation_access_error">
             <field name="name">access_error</field>
             <field name="lang">en_US</field>

trytond/model/modelstorage.py

@@ -1066,6 +1066,37 @@ class ModelStorage(Model):
                                     user, field_name, context=context),
                                 context=context)
 
+            # validate digits
+            if hasattr(field, 'digits') and field.digits:
+                if is_pyson(field.digits):
+                    pyson_digits = PYSONEncoder().encode(field.digits)
+                    ctx = context.copy()
+                    ctx.update(ctx_pref)
+                    for record in records:
+                        env = EvalEnvironment(record, self)
+                        env.update(ctx)
+                        env['current_date'] = datetime.datetime.today()
+                        env['time'] = time
+                        env['context'] = context
+                        env['active_id'] = record.id
+                        digits = PYSONDecoder(env).decode(pyson_digits)
+                        if not round(record[field_name], digits[1]) == \
+                                float(record[field_name]):
+                            self.raise_user_error(cursor,
+                                    'digits_validation_record',
+                                    error_args=self._get_error_args(cursor,
+                                        user, field_name, context=context),
+                                    context=context)
+                else:
+                    for record in records:
+                        if not round(record[field_name], field.digits[1]) == \
+                                float(record[field_name]):
+                            self.raise_user_error(cursor,
+                                    'digits_validation_record',
+                                    error_args=self._get_error_args(cursor,
+                                        user, field_name, context=context),
+                                    context=context)
+
     def _clean_defaults(self, defaults):
         vals = {}
         for field in defaults.keys():

trytond/test/test.py

@@ -1,8 +1,10 @@
 #This file is part of Tryton.  The COPYRIGHT file at the top level of
 #this repository contains the full copyright notices and license terms.
 
-from trytond.model import ModelSQL, fields
 import datetime
+from decimal import Decimal
+from trytond.model import ModelSQL, fields
+from trytond.pyson import Eval
 
 
 class Boolean(ModelSQL):
@@ -94,6 +96,17 @@ class FloatRequired(ModelSQL):
 FloatRequired()
 
 
+class FloatDigits(ModelSQL):
+    'Float Digits'
+    _name = 'test.float_digits'
+    _description = __doc__
+    digits = fields.Integer('Digits')
+    float = fields.Float(string='Float', help='Test float',
+            required=False, digits=(16, Eval('digits', 2)))
+
+FloatDigits()
+
+
 class Numeric(ModelSQL):
     'Numeric'
     _name = 'test.numeric'
@@ -112,7 +125,7 @@ class NumericDefault(ModelSQL):
             required=False)
 
     def default_numeric(self, cursor, user, context=None):
-        return 5.5
+        return Decimal('5.5')
 
 NumericDefault()
 
@@ -127,6 +140,17 @@ class NumericRequired(ModelSQL):
 NumericRequired()
 
 
+class NumericDigits(ModelSQL):
+    'Numeric Digits'
+    _name = 'test.numeric_digits'
+    _description = __doc__
+    digits = fields.Integer('Digits')
+    numeric = fields.Numeric(string='Numeric', help='Test numeric',
+            required=False, digits=(16, Eval('digits', 2)))
+
+NumericDigits()
+
+
 class Char(ModelSQL):
     'Char'
     _name = 'test.char'

trytond/tests/test_fields.py

@@ -29,10 +29,12 @@ class FieldsTestCase(unittest.TestCase):
         self.float = POOL.get('test.float')
         self.float_default = POOL.get('test.float_default')
         self.float_required = POOL.get('test.float_required')
+        self.float_digits = POOL.get('test.float_digits')
 
         self.numeric = POOL.get('test.numeric')
         self.numeric_default = POOL.get('test.numeric_default')
         self.numeric_required = POOL.get('test.numeric_required')
+        self.numeric_digits = POOL.get('test.numeric_digits')
 
         self.char = POOL.get('test.char')
         self.char_default = POOL.get('test.char_default')
@@ -540,6 +542,35 @@ class FieldsTestCase(unittest.TestCase):
                 CONTEXT)
         self.assert_(float5['float'] == 0)
 
+        float6_id = self.float_digits.create(cursor, USER, {
+            'digits': 1,
+            'float': 1.1,
+            }, CONTEXT)
+        self.assert_(float6_id)
+
+        self.failUnlessRaises(Exception, self.float_digits.create, cursor,
+                USER, {
+                    'digits': 1,
+                    'float': 1.11,
+                    }, CONTEXT)
+
+        self.failUnlessRaises(Exception, self.float_digits.write, cursor,
+                USER, float6_id, {
+                    'float': 1.11,
+                    }, CONTEXT)
+
+        self.failUnlessRaises(Exception, self.float_digits.write, cursor,
+                USER, float6_id, {
+                    'digits': 0,
+                    }, CONTEXT)
+
+        float7_id = self.float.create(cursor, USER, {
+            'float': 0.1234567890123456789,
+            }, CONTEXT)
+
+        float7 = self.float.read(cursor, USER, float7_id, ['float'], CONTEXT)
+        self.assert_(float7['float'] == 0.1234567890123456789)
+
         cursor.rollback()
         cursor.close()
 
@@ -736,6 +767,37 @@ class FieldsTestCase(unittest.TestCase):
                 ['numeric'], CONTEXT)
         self.assert_(numeric5['numeric'] == Decimal('0'))
 
+        numeric6_id = self.numeric_digits.create(cursor, USER, {
+            'digits': 1,
+            'numeric': Decimal('1.1'),
+            }, CONTEXT)
+        self.assert_(numeric6_id)
+
+        self.failUnlessRaises(Exception, self.numeric_digits.create, cursor,
+                USER, {
+                    'digits': 1,
+                    'numeric': Decimal('1.11'),
+                    }, CONTEXT)
+
+        self.failUnlessRaises(Exception, self.numeric_digits.write, cursor,
+                USER, numeric6_id, {
+                    'numeric': Decimal('1.11'),
+                    }, CONTEXT)
+
+        self.failUnlessRaises(Exception, self.numeric_digits.write, cursor,
+                USER, numeric6_id, {
+                    'digits': 0,
+                    }, CONTEXT)
+
+        numeric7_id = self.numeric.create(cursor, USER, {
+            'numeric': Decimal('0.1234567890123456789'),
+            }, CONTEXT)
+
+        numeric7 = self.numeric.read(cursor, USER, numeric7_id, ['numeric'],
+                CONTEXT)
+        self.assert_(numeric7['numeric'] ==
+                Decimal('0.1234567890123456789'))
+
         cursor.rollback()
         cursor.close()
 

trytond/tools/misc.py

@@ -18,6 +18,7 @@ except ImportError:
     import StringIO
 import dis
 import datetime
+from decimal import Decimal
 
 def find_in_path(name):
     if os.name == "nt":
@@ -585,4 +586,6 @@ def safe_eval(source, data=None):
         'locals': locals,
         'bool': bool,
         'dict': dict,
+        'round': round,
+        'Decimal': Decimal,
         }}, data)