diff --git a/CHANGELOG b/CHANGELOG index 58ff6362ac..941734b040 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +* Use python-sql +* Add convert_domain method on Fields +* Add sql_format and sql_type methods on Fields +* Allow to return a full domain for Function.searcher +* Replace static backend by dynamic get method +* Replace order_field by order_ method * Allow field One2One in check_recursion * Remove the default order on MPTT * Add grouped attribute to data tag diff --git a/doc/ref/models/fields.rst b/doc/ref/models/fields.rst index d40f652670..451392fe12 100644 --- a/doc/ref/models/fields.rst +++ b/doc/ref/models/fields.rst @@ -122,15 +122,6 @@ client will also read these fields even if they are not defined on the view. :attr:`Field.depends` is used per example to ensure that :class:`~trytond.pyson.PYSON` statement could be evaluated. -``order_field`` ---------------- - -.. attribute:: Field.order_field - -The name of a substitute field on which the ordering of records must be done -instead of this one. -This is often used to allow ordering on :class:`Function` fields. - ``context`` ----------- @@ -153,11 +144,51 @@ Define how the field must be loaded: ``lazy`` or ``eager``. The name of the field. +Instance methods: + +.. method:: Field.convert_domain(domain, tables, Model) + + Convert the simple :ref:`domain ` clause into a SQL + expression or a new domain. + +Where ``tables`` is a nested dictionary containing the existing joins (and it +could be updated to add new joins):: + + { + None: (, None), + 'party': { + None: (
, ), + 'addresses': { + None: (
, ), + }, + }, + } + +.. method:: Field.sql_format(value) + + Convert the value to use as parameter of SQL queries. + +.. method:: Field.sql_type() + + Return the namedtuple('SQLType', 'base type') which defines the SQL type to + use for creation and casting. + Default value ============= See :ref:`default value ` +Ordering +======== + +A class method could be defined for each field which must return a list of SQL +expression on which to order instead of the field. +The method signature is:: + + order_(tables) + +Where ``tables`` is a nested dictionary, see :meth:`~Field.convert_domain`. + Field types =========== @@ -289,6 +320,13 @@ A date and time, represented in Python by a ``datetime.datetime`` instance. the time part of the field. The default value is `%H:%M:%S`. The value can be replaced by a :class:`~trytond.pyson.PYSON` statement. +Timestamp +--------- + +.. class:: Timestamp(string[, \**options]) + +A timestamp, represented in Python by a ``datetime.datetime`` instance. + Time ---- @@ -635,7 +673,8 @@ A function field can emulate any other given `field`. where `name` is the name of the field and `clause` is a :ref:`domain clause `. - It must return a list of :ref:`domain ` clauses. + It must return a list of :ref:`domain ` clauses but the + ``operand`` can be a SQL query. Instance methods: diff --git a/doc/ref/models/models.rst b/doc/ref/models/models.rst index 1816ce6cda..54b092c802 100644 --- a/doc/ref/models/models.rst +++ b/doc/ref/models/models.rst @@ -391,10 +391,19 @@ Class attributes are: Class methods: +.. classmethod:: ModelSQL.__table__() + + Return a SQL Table instance for the Model. + .. classmethod:: ModelSQL.table_query() Could be overrided to use a custom SQL query instead of a table of the - database. It should return a tuple containing SQL query and arguments. + database. It should return a SQL FromItem. + +.. classmethod:: ModelStorage.search(domain[, offset[, limit[, order[, count[, query]]]]]) + + Return a list of records that match the :ref:`domain ` or + the sql query if query is True. .. classmethod:: ModelSQL.search_domain(domain[, active_test]) diff --git a/doc/topics/install.rst b/doc/topics/install.rst index 4e18982eed..b18da81b04 100644 --- a/doc/topics/install.rst +++ b/doc/topics/install.rst @@ -12,6 +12,7 @@ Prerequisites * relatorio 0.2.0 or later (http://relatorio.openhex.org/) * python-dateutil (http://labix.org/python-dateutil) * polib (https://bitbucket.org/izi/polib/wiki/Home) + * python-sql (http://code.google.com/p/python-sql/) * Optional: psycopg 2 or later (http://www.initd.org/) * Optional: MySQL-python (http://sourceforge.net/projects/mysql-python/) * Optional: pywebdav 0.9.8 or later (http://code.google.com/p/pywebdav/) diff --git a/setup.py b/setup.py index 5d4e8c38ed..dfb43a3b44 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ setup(name=PACKAGE, 'Genshi', 'python-dateutil', 'polib', + 'python-sql', ], extras_require={ 'PostgreSQL': ['psycopg2 >= 2.0'], diff --git a/trytond/backend/__init__.py b/trytond/backend/__init__.py index 18c1a6ba87..56202911f2 100644 --- a/trytond/backend/__init__.py +++ b/trytond/backend/__init__.py @@ -1,15 +1,15 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +import sys from trytond.config import CONFIG -db_type = CONFIG['db_type'] -if db_type == 'sqlite': - from .sqlite import * -elif db_type == 'mysql': - from .mysql import * -elif db_type == 'postgresql': - from .postgresql import * -else: - raise ValueError('db_type: "%s" is not supported' % db_type) +__all__ = ['get'] + +def get(name): + db_type = CONFIG['db_type'] + modname = 'trytond.backend.%s' % db_type + __import__(modname) + module = sys.modules[modname] + return getattr(module, name) diff --git a/trytond/backend/database.py b/trytond/backend/database.py index f0d31bfb72..c80e370210 100644 --- a/trytond/backend/database.py +++ b/trytond/backend/database.py @@ -10,6 +10,7 @@ class DatabaseInterface(object): ''' Define generic interface for database connection ''' + flavor = None def __new__(cls, database_name=''): return object.__new__(cls) @@ -101,7 +102,6 @@ class CursorInterface(object): ''' Define generic interface for database cursor ''' - sql_log = False IN_MAX = 1000 def __init__(self): @@ -210,17 +210,6 @@ class CursorInterface(object): ''' raise NotImplementedError - def limit_clause(self, select, limit=None, offset=None): - ''' - Return SELECT queries with limit and offset - - :param select: the SELECT query string - :param limit: the limit - :param offset: the offset - :return: a string - ''' - raise NotImplementedError - def update_auto_increment(self, table, value): ''' Update auto_increment value of table diff --git a/trytond/backend/fields.py b/trytond/backend/fields.py deleted file mode 100644 index a591310d06..0000000000 --- a/trytond/backend/fields.py +++ /dev/null @@ -1,200 +0,0 @@ -#This file is part of Tryton. The COPYRIGHT file at the top level of -#this repository contains the full copyright notices and license terms. -import datetime -from decimal import Decimal -import hashlib -try: - import simplejson as json -except ImportError: - import json - - -class Field(object): - - @staticmethod - def sql_format(value): - return value - - @staticmethod - def sql_type(field): - raise NotImplementedError - - -class Boolean(Field): - pass - - -class Integer(Field): - pass - - -class BigInteger(Integer): - pass - - -class Char(Field): - - @staticmethod - def sql_format(value): - if value is None: - return None - elif isinstance(value, str): - return unicode(value, 'utf-8') - assert isinstance(value, unicode) - return value - - -class Sha(Field): - - @staticmethod - def sql_format(value): - if value is not None: - if isinstance(value, unicode): - value = value.encode('utf-8') - value = hashlib.sha1(value).hexdigest() - return Char.sql_format(value) - - -class Text(Char): - pass - - -class Float(Field): - pass - - -class Numeric(Float): - - @staticmethod - def sql_format(value): - if value is None: - return value - if isinstance(value, (int, long)): - value = Decimal(str(value)) - assert isinstance(value, Decimal) - return value - - -class Date(Field): - - @staticmethod - def sql_format(value): - if not value: - return None - if isinstance(value, basestring): - year, month, day = map(int, value.split("-", 2)) - return datetime.date(year, month, day) - - assert(isinstance(value, datetime.date)) - # Allow datetime with min time for XML-RPC - # datetime must be tested separately because datetime is a - # subclass of date - assert(not isinstance(value, datetime.datetime) - or value.time() == datetime.time()) - return value - - -class DateTime(Field): - - @staticmethod - def sql_format(value): - if not value: - return None - if isinstance(value, basestring): - datepart, timepart = value.split(" ") - year, month, day = map(int, datepart.split("-", 2)) - hours, minutes, seconds = map(int, timepart.split(":")) - return datetime.datetime(year, month, day, hours, minutes, seconds) - assert(isinstance(value, datetime.datetime)) - return value.replace(microsecond=0) - - -class Timestamp(Field): - - @staticmethod - def sql_format(value): - if not value: - return None - if isinstance(value, basestring): - datepart, timepart = value.split(" ") - year, month, day = map(int, datepart.split("-", 2)) - timepart_full = timepart.split(".", 1) - hours, minutes, seconds = map(int, timepart_full[0].split(":")) - if len(timepart_full) == 2: - microseconds = int(timepart_full[1]) - else: - microseconds = 0 - return datetime.datetime(year, month, day, hours, minutes, seconds, - microseconds) - assert(isinstance(value, datetime.datetime)) - return value - - -class Time(Field): - - @staticmethod - def sql_format(value): - if not value: - return None - if isinstance(value, basestring): - hours, minutes, seconds = map(int, value.split(":")) - return datetime.time(hours, minutes, seconds) - assert(isinstance(value, datetime.time)) - return value.replace(microsecond=0) - - -class Binary(Field): - pass - - -class Selection(Char): - pass - - -class Reference(Char): - - @staticmethod - def sql_format(value): - if not isinstance(value, basestring): - try: - value = '%s,%s' % tuple(value) - except TypeError: - pass - return Char.sql_format(value) - - -class Many2One(Field): - - @staticmethod - def sql_format(value): - if value is None: - return None - assert value is not False - return int(value) - - -class One2Many(Field): - pass - - -class Many2Many(Field): - pass - - -class Function(Field): - pass - - -class Property(Function): - pass - - -class Dict(Field): - - @staticmethod - def sql_format(value): - from trytond.protocols.jsonrpc import JSONEncoder - if value is None: - return None - assert isinstance(value, dict) - return json.dumps(value, cls=JSONEncoder) diff --git a/trytond/backend/mysql/__init__.py b/trytond/backend/mysql/__init__.py index 71229c07be..7f49e2365c 100644 --- a/trytond/backend/mysql/__init__.py +++ b/trytond/backend/mysql/__init__.py @@ -2,5 +2,4 @@ #this repository contains the full copyright notices and license terms. from .database import * -from .fields import * from .table import * diff --git a/trytond/backend/mysql/database.py b/trytond/backend/mysql/database.py index 724b668681..792998cefd 100644 --- a/trytond/backend/mysql/database.py +++ b/trytond/backend/mysql/database.py @@ -7,34 +7,72 @@ import MySQLdb.cursors import MySQLdb.converters from MySQLdb import IntegrityError as DatabaseIntegrityError from MySQLdb import OperationalError as DatabaseOperationalError -import logging import os -import re import time import tempfile - -QUOTE_SEPARATION = re.compile(r"(.*?)('.*?')", re.DOTALL) -EXTRACT_EPOCH_PATTERN = re.compile(r'EXTRACT\s*\(\s*EPOCH\s+FROM', - re.I) -CAST_VARCHAR_PATTERN = re.compile(r' AS VARCHAR\)', re.I) -CAST_INTEGER_PATTERN = re.compile(r' AS (INTEGER|BIGINT)\)', re.I) -SPLIT_PART_LEFT_PATTERN = re.compile(r'SPLIT_PART\((.*?),', re.I) -SPLIT_PART_RIGHT_PATTERN = re.compile(r'^,(\d)', re.I) +from sql import Flavor, Expression +from sql.functions import Extract, Overlay, CharLength -def _replace_split_part_right(mobj): - pos = int(mobj.group(1)) - if pos not in (1, 2): - raise Exception('SPLIT_PART is only partially implemented for MySQL') - if pos == 2: - return ', -1' - return ', 1' +class MySQLExtract(Extract): + + def is_epoch(self): + return self.args[0].upper() == 'EPOCH' + + def __str__(self): + if self.is_epoch(): + return 'UNIX_TIMESTAMP(%s)' % self._format(self.args[1]) + return super(MySQLExtract, self).__str__() + + @property + def params(self): + if self.is_epoch(): + arg = self.args[1] + if isinstance(arg, Expression): + return arg.params + else: + return (arg,) + return super(MySQLExtract, self).params + + +class MySQLOverlay(Overlay): + + @property + def mysql_args(self): + if len(self.args) == 3: + string, placing, from_ = self.args + for_ = CharLength(placing) + else: + string, placing, from_, for_ = self.args + return (string, from_, placing, string, from_, for_) + + def __str__(self): + return ('CONCAT(SUBSTRING(%s FROM 1 FOR %s), %s, ' + 'SUBSTRING(%s FROM %s + 1 + %s))' + % tuple(map(self._format, self.mysql_args))) + + @property + def params(self): + p = () + for arg in self.mysql_args: + if isinstance(arg, Expression): + p += arg.params + else: + p += (arg,) + return p + + +MAPPING = { + Extract: MySQLExtract, + Overlay: MySQLOverlay, + } class Database(DatabaseInterface): _list_cache = None _list_cache_timestamp = None + flavor = Flavor(max_limit=18446744073709551610, function_mapping=MAPPING) def connect(self): return self @@ -234,38 +272,10 @@ class Cursor(CursorInterface): return getattr(self.cursor, name) def execute(self, sql, params=None): - buf = "" - split_part_found = False - for nquote, quote in QUOTE_SEPARATION.findall(sql + "''"): - nquote = nquote.replace('ilike', 'like') - nquote = re.sub(EXTRACT_EPOCH_PATTERN, r'UNIX_TIMESTAMP(', - nquote) - nquote = re.sub(CAST_VARCHAR_PATTERN, r' AS CHAR)', - nquote) - nquote = re.sub(CAST_INTEGER_PATTERN, r' AS SIGNED INTEGER)', - nquote) - if split_part_found: - nquote = re.sub(SPLIT_PART_RIGHT_PATTERN, - _replace_split_part_right, nquote) - split_part_found = False - nquote, split_part_found = re.subn(SPLIT_PART_LEFT_PATTERN, - r'SUBSTRING_INDEX(\1, ', nquote) - - buf += nquote + quote - sql = buf[:-2] - try: - if params: - res = self.cursor.execute(sql, params) - else: - res = self.cursor.execute(sql) - except Exception, exception: - logger = logging.getLogger('sql') - logger.error('Wrong SQL: ' + sql + ' ' + str(params)) - if isinstance(exception, MySQLdb.Error): - exception.args = exception.args[1:] - raise exception - raise - return res + if params: + return self.cursor.execute(sql, params) + else: + return self.cursor.execute(sql) def close(self, close=False): self.cursor.close() diff --git a/trytond/backend/mysql/fields.py b/trytond/backend/mysql/fields.py deleted file mode 100644 index 8a536d7be6..0000000000 --- a/trytond/backend/mysql/fields.py +++ /dev/null @@ -1,156 +0,0 @@ -#This file is part of Tryton. The COPYRIGHT file at the top level of -#this repository contains the full copyright notices and license terms. -from trytond.backend import fields - - -class Boolean(fields.Boolean): - - @staticmethod - def sql_type(field): - return ('bool', 'bool') - - @staticmethod - def sql_format(value): - return value - - -class Integer(fields.Integer): - - @staticmethod - def sql_type(field): - return ('bigint', 'bigint') - - -class BigInteger(fields.BigInteger): - - @staticmethod - def sql_type(field): - return ('bigint', 'bigint') - - -class Char(fields.Char): - - @staticmethod - def sql_type(field): - if isinstance(field.size, int): - return ('varchar', 'varchar(%d)' % (field.size,)) - return ('varchar', 'varchar(255)') - - -class Sha(fields.Sha): - - @staticmethod - def sql_type(field): - return ('varchar', 'varchar(40)') - - -class Text(fields.Text): - - @staticmethod - def sql_type(field): - return ('text', 'text') - - -class Float(fields.Float): - - @staticmethod - def sql_type(field): - return ('double', 'double(255, 15)') - - -class Numeric(fields.Numeric): - - @staticmethod - def sql_type(field): - return ('decimal', 'decimal(65, 30)') - - -class Date(fields.Date): - - @staticmethod - def sql_type(field): - return ('date', 'date') - - -class DateTime(fields.DateTime): - - @staticmethod - def sql_type(field): - return ('timestamp', 'timestamp null') - - -class Timestamp(fields.Timestamp): - - @staticmethod - def sql_type(field): - return ('timestamp', 'timestamp null') - - -class Time(fields.Time): - - @staticmethod - def sql_type(field): - return ('time', 'time') - - -class Binary(fields.Binary): - - @staticmethod - def sql_format(value): - return value or None - - @staticmethod - def sql_type(field): - return ('longblob', 'longblob') - - -class Selection(fields.Selection): - - @staticmethod - def sql_type(field): - return ('varchar', 'varchar(255)') - - -class Reference(fields.Reference): - - @staticmethod - def sql_type(field): - return ('varchar', 'varchar(255)') - - -class Many2One(fields.Many2One): - - @staticmethod - def sql_type(field): - return ('bigint', 'bigint') - - -class Dict(fields.Dict): - - @staticmethod - def sql_type(field): - return ('text', 'text') - -FIELDS = { - 'boolean': Boolean, - 'integer': Integer, - 'biginteger': BigInteger, - 'char': Char, - 'sha': Sha, - 'text': Text, - 'float': Float, - 'numeric': Numeric, - 'date': Date, - 'datetime': DateTime, - 'timestamp': Timestamp, - 'time': Time, - 'binary': Binary, - 'selection': Selection, - 'reference': Reference, - 'many2one': Many2One, - 'one2many': fields.One2Many, - 'many2many': fields.Many2Many, - 'function': fields.Function, - 'property': fields.Property, - 'dict': Dict, -} diff --git a/trytond/backend/mysql/table.py b/trytond/backend/mysql/table.py index 4a5dc7b098..3fc878fd51 100644 --- a/trytond/backend/mysql/table.py +++ b/trytond/backend/mysql/table.py @@ -166,6 +166,11 @@ class TableHandler(TableHandlerInterface): if not migrate: return base_type = column_type[0].lower() + convert = { + 'char': 'varchar', + 'signed integer': 'bigint', + } + base_type = convert.get(base_type, base_type) if base_type != self._columns[column_name]['typname']: if (self._columns[column_name]['typname'], base_type) in ( ('varchar', 'text'), diff --git a/trytond/backend/postgresql/__init__.py b/trytond/backend/postgresql/__init__.py index 71229c07be..7f49e2365c 100644 --- a/trytond/backend/postgresql/__init__.py +++ b/trytond/backend/postgresql/__init__.py @@ -2,5 +2,4 @@ #this repository contains the full copyright notices and license terms. from .database import * -from .fields import * from .table import * diff --git a/trytond/backend/postgresql/database.py b/trytond/backend/postgresql/database.py index 16639e68d9..2c55688749 100644 --- a/trytond/backend/postgresql/database.py +++ b/trytond/backend/postgresql/database.py @@ -21,6 +21,10 @@ import os if os.name == 'posix': import pwd from decimal import Decimal +from sql import Flavor + +__all__ = ['Database', 'DatabaseIntegrityError', 'DatabaseOperationalError', + 'Cursor'] RE_FROM = re.compile('.* from "?([a-zA-Z_0-9]+)"?.*$') RE_INTO = re.compile('.* into "?([a-zA-Z_0-9]+)"?.*$') @@ -34,6 +38,7 @@ class Database(DatabaseInterface): _list_cache = None _list_cache_timestamp = None _version_cache = {} + flavor = Flavor(ilike=True) def __new__(cls, database_name='template1'): if database_name in cls._databases: @@ -298,51 +303,16 @@ class Cursor(CursorInterface): return getattr(self.cursor, name) def execute(self, sql, params=None): - if self.sql_log: - now = time.time() - try: if params: - res = self.cursor.execute(sql, params) + return self.cursor.execute(sql, params) else: - res = self.cursor.execute(sql) + return self.cursor.execute(sql) except Exception: - logger = logging.getLogger('sql') - logger.error('Wrong SQL: ' + (self.cursor.query or '')) + print sql, params raise - if self.sql_log: - res_from = RE_FROM.match(sql.lower()) - if res_from: - self.sql_from_log.setdefault(res_from.group(1), [0, 0]) - self.sql_from_log[res_from.group(1)][0] += 1 - self.sql_from_log[res_from.group(1)][1] += time.time() - now - self.count['from'] += 1 - res_into = RE_INTO.match(sql.lower()) - if res_into: - self.sql_into_log.setdefault(res_into.group(1), [0, 0]) - self.sql_into_log[res_into.group(1)][0] += 1 - self.sql_into_log[res_into.group(1)][1] += time.time() - now - self.count['into'] += 1 - return res - - def _print_log(self, sql_type='from'): - logger = logging.getLogger('sql') - logger.info("SQL LOG %s:" % (sql_type,)) - if sql_type == 'from': - logs = self.sql_from_log.items() - else: - logs = self.sql_into_log.items() - logs.sort(lambda x, y: cmp(x[1][1], y[1][1])) - amount = 0 - for log in logs: - logger.info("table:%s:%f/%d" % (log[0], log[1][1], log[1][0])) - amount += log[1][1] - logger.info("SUM:%s/%d" % (amount, self.count[sql_type])) def close(self, close=False): - if self.sql_log: - self._print_log('from') - self._print_log('into') self.cursor.close() self.rollback() self._connpool.putconn(self._conn, close=close) diff --git a/trytond/backend/postgresql/fields.py b/trytond/backend/postgresql/fields.py deleted file mode 100644 index c4ba85c206..0000000000 --- a/trytond/backend/postgresql/fields.py +++ /dev/null @@ -1,153 +0,0 @@ -#This file is part of Tryton. The COPYRIGHT file at the top level of -#this repository contains the full copyright notices and license terms. -import psycopg2 -from trytond.backend import fields - - -class Boolean(fields.Boolean): - - @staticmethod - def sql_type(field): - return ('bool', 'bool') - - -class Integer(fields.Integer): - - @staticmethod - def sql_type(field): - return ('int4', 'int4') - - -class BigInteger(fields.BigInteger): - - @staticmethod - def sql_type(field): - return ('int8', 'int8') - - -class Char(fields.Char): - - @staticmethod - def sql_type(field): - if isinstance(field.size, int): - return ('varchar', 'varchar(%d)' % (field.size,)) - return ('varchar', 'varchar') - - -class Sha(fields.Sha): - - @staticmethod - def sql_type(field): - return ('varchar', 'varchar(40)') - - -class Text(fields.Text): - - @staticmethod - def sql_type(field): - return ('text', 'text') - - -class Float(fields.Float): - - @staticmethod - def sql_type(field): - return ('float8', 'float8') - - -class Numeric(fields.Numeric): - - @staticmethod - def sql_type(field): - return ('numeric', 'numeric') - - -class Date(fields.Date): - - @staticmethod - def sql_type(field): - return ('date', 'date') - - -class DateTime(fields.DateTime): - - @staticmethod - def sql_type(field): - return ('timestamp', 'timestamp(0)') - - -class Timestamp(fields.Timestamp): - - @staticmethod - def sql_type(field): - return ('timestamp', 'timestamp(6)') - - -class Time(fields.Time): - - @staticmethod - def sql_type(field): - return ('time', 'time') - - -class Binary(fields.Binary): - - @staticmethod - def sql_format(value): - return value and psycopg2.Binary(value) or None - - @staticmethod - def sql_type(field): - return ('bytea', 'bytea') - - -class Selection(fields.Selection): - - @staticmethod - def sql_type(field): - return ('varchar', 'varchar') - - -class Reference(fields.Reference): - - @staticmethod - def sql_type(field): - return ('varchar', 'varchar') - - -class Many2One(fields.Many2One): - - @staticmethod - def sql_type(field): - return ('int4', 'int4') - - -class Dict(fields.Dict): - - @staticmethod - def sql_type(field): - return ('text', 'text') - -FIELDS = { - 'boolean': Boolean, - 'integer': Integer, - 'biginteger': BigInteger, - 'char': Char, - 'sha': Sha, - 'text': Text, - 'float': Float, - 'numeric': Numeric, - 'date': Date, - 'datetime': DateTime, - 'timestamp': Timestamp, - 'time': Time, - 'binary': Binary, - 'selection': Selection, - 'reference': Reference, - 'many2one': Many2One, - 'one2many': fields.One2Many, - 'many2many': fields.Many2Many, - 'function': fields.Function, - 'property': fields.Property, - 'dict': Dict, -} diff --git a/trytond/backend/postgresql/table.py b/trytond/backend/postgresql/table.py index 1c3a2e9fb3..be4478fbd2 100644 --- a/trytond/backend/postgresql/table.py +++ b/trytond/backend/postgresql/table.py @@ -4,6 +4,8 @@ from trytond.backend.table import TableHandlerInterface import logging +__all__ = ['TableHandler'] + class TableHandler(TableHandlerInterface): diff --git a/trytond/backend/sqlite/__init__.py b/trytond/backend/sqlite/__init__.py index 71229c07be..7f49e2365c 100644 --- a/trytond/backend/sqlite/__init__.py +++ b/trytond/backend/sqlite/__init__.py @@ -2,5 +2,4 @@ #this repository contains the full copyright notices and license terms. from .database import * -from .fields import * from .table import * diff --git a/trytond/backend/sqlite/database.py b/trytond/backend/sqlite/database.py index 0f2123f5a2..b61a99dcdc 100644 --- a/trytond/backend/sqlite/database.py +++ b/trytond/backend/sqlite/database.py @@ -3,7 +3,6 @@ from trytond.backend.database import DatabaseInterface, CursorInterface from trytond.config import CONFIG import os -import re from decimal import Decimal import datetime import time @@ -21,49 +20,56 @@ except ImportError: import sqlite3 as sqlite from sqlite3 import IntegrityError as DatabaseIntegrityError from sqlite3 import OperationalError as DatabaseOperationalError -QUOTE_SEPARATION = re.compile(r"(.*?)('.*?')", re.DOTALL) -EXTRACT_PATTERN = re.compile(r'EXTRACT\s*\(\s*(\S*)\s+FROM', re.I) -POSITION_PATTERN = re.compile(r'POSITION\s*\(([^\)]*)\s+IN', re.I) +from sql import Flavor, Table +from sql.functions import Function, Extract, Position, Now, Substring, Overlay + +__all__ = ['Database', 'DatabaseIntegrityError', 'DatabaseOperationalError', + 'Cursor'] -def extract(lookup_type, date): - if date is None: - return None - if len(date) == 10: - year, month, day = map(int, date.split('-')) - date = datetime.date(year, month, day) - else: - datepart, timepart = date.split(" ") - year, month, day = map(int, datepart.split("-")) - timepart_full = timepart.split(".") - hours, minutes, seconds = map(int, timepart_full[0].split(":")) - if len(timepart_full) == 2: - microseconds = int(timepart_full[1]) +class SQLiteExtract(Function): + __slots__ = () + _function = 'EXTRACT' + + @staticmethod + def extract(lookup_type, date): + if date is None: + return None + if len(date) == 10: + year, month, day = map(int, date.split('-')) + date = datetime.date(year, month, day) else: - microseconds = 0 - date = datetime.datetime(year, month, day, hours, minutes, seconds, - microseconds) - if lookup_type.lower() == 'century': - return date.year / 100 + (date.year % 100 and 1 or 0) - elif lookup_type.lower() == 'decade': - return date.year / 10 - elif lookup_type.lower() == 'dow': - return (date.weekday() + 1) % 7 - elif lookup_type.lower() == 'doy': - return date.timetuple().tm_yday - elif lookup_type.lower() == 'epoch': - return int(time.mktime(date.timetuple())) - elif lookup_type.lower() == 'microseconds': - return date.microsecond - elif lookup_type.lower() == 'millennium': - return date.year / 1000 + (date.year % 1000 and 1 or 0) - elif lookup_type.lower() == 'milliseconds': - return date.microsecond / 1000 - elif lookup_type.lower() == 'quarter': - return date.month / 4 + 1 - elif lookup_type.lower() == 'week': - return date.isocalendar()[1] - return getattr(date, lookup_type.lower()) + datepart, timepart = date.split(" ") + year, month, day = map(int, datepart.split("-")) + timepart_full = timepart.split(".") + hours, minutes, seconds = map(int, timepart_full[0].split(":")) + if len(timepart_full) == 2: + microseconds = int(timepart_full[1]) + else: + microseconds = 0 + date = datetime.datetime(year, month, day, hours, minutes, seconds, + microseconds) + if lookup_type.lower() == 'century': + return date.year / 100 + (date.year % 100 and 1 or 0) + elif lookup_type.lower() == 'decade': + return date.year / 10 + elif lookup_type.lower() == 'dow': + return (date.weekday() + 1) % 7 + elif lookup_type.lower() == 'doy': + return date.timetuple().tm_yday + elif lookup_type.lower() == 'epoch': + return int(time.mktime(date.timetuple())) + elif lookup_type.lower() == 'microseconds': + return date.microsecond + elif lookup_type.lower() == 'millennium': + return date.year / 1000 + (date.year % 1000 and 1 or 0) + elif lookup_type.lower() == 'milliseconds': + return date.microsecond / 1000 + elif lookup_type.lower() == 'quarter': + return date.month / 4 + 1 + elif lookup_type.lower() == 'week': + return date.isocalendar()[1] + return getattr(date, lookup_type.lower()) def date_trunc(_type, date): @@ -88,23 +94,57 @@ def split_part(text, delimiter, count): return (text.split(delimiter) + [''] * (count - 1))[count - 1] -def position(substring, string): - if string is None: - return - try: - return string.index(substring) + 1 - except ValueError: - return 0 +class SQLitePosition(Function): + __slots__ = () + _function = 'POSITION' + + @staticmethod + def position(substring, string): + if string is None: + return + try: + return string.index(substring) + 1 + except ValueError: + return 0 def replace(text, pattern, replacement): return str(text).replace(pattern, replacement) +def now(): + return datetime.datetime.now().isoformat(' ') + + +class SQLiteSubstring(Function): + __slots__ = () + _function = 'SUBSTR' + + +class SQLiteOverlay(Function): + __slots__ = () + _function = 'OVERLAY' + + @staticmethod + def overlay(string, placing_string, from_, for_=None): + if for_ is None: + for_ = len(placing_string) + return string[:from_ - 1] + placing_string + string[from_ - 1 + for_:] + + +MAPPING = { + Extract: SQLiteExtract, + Position: SQLitePosition, + Substring: SQLiteSubstring, + Overlay: SQLiteOverlay, + } + + class Database(DatabaseInterface): _local = threading.local() _conn = None + flavor = Flavor(paramstyle='qmark', function_mapping=MAPPING) def __new__(cls, database_name=':memory:'): if (database_name == ':memory:' @@ -129,12 +169,15 @@ class Database(DatabaseInterface): if self._conn is not None: return self self._conn = sqlite.connect(path, detect_types=sqlite.PARSE_DECLTYPES) - self._conn.create_function('extract', 2, extract) + self._conn.create_function('extract', 2, SQLiteExtract.extract) self._conn.create_function('date_trunc', 2, date_trunc) self._conn.create_function('split_part', 3, split_part) - self._conn.create_function('position', 2, position) + self._conn.create_function('position', 2, SQLitePosition.position) + self._conn.create_function('overlay', 3, SQLiteOverlay.overlay) + self._conn.create_function('overlay', 4, SQLiteOverlay.overlay) if sqlite.sqlite_version_info < (3, 3, 14): self._conn.create_function('replace', 3, replace) + self._conn.create_function('now', 0, now) return self def cursor(self, autocommit=False, readonly=False): @@ -232,22 +275,28 @@ class Database(DatabaseInterface): if (len(line) > 0) and (not line.isspace()): cursor.execute(line) + ir_module = Table('ir_module_module') + ir_module_dependency = Table('ir_module_module_dependency') for module in ('ir', 'res', 'webdav'): state = 'uninstalled' if module in ('ir', 'res'): state = 'to install' info = get_module_info(module) - cursor.execute('INSERT INTO ir_module_module ' - '(create_uid, create_date, name, state) ' - 'VALUES (%s, %s, %s, %s)', - (0, datetime.datetime.now(), module, state)) + insert = ir_module.insert( + [ir_module.create_uid, ir_module.create_date, ir_module.name, + ir_module.state], + [[0, Now(), module, state]]) + cursor.execute(*insert) cursor.execute('SELECT last_insert_rowid()') module_id, = cursor.fetchone() for dependency in info.get('depends', []): - cursor.execute('INSERT INTO ir_module_module_dependency ' - '(create_uid, create_date, module, name) ' - 'VALUES (%s, %s, %s, %s) ', - (0, datetime.datetime.now(), module_id, dependency)) + insert = ir_module_dependency.insert( + [ir_module_dependency.create_uid, + ir_module_dependency.create_date, + ir_module_dependency.module, ir_module_dependency.name + ], + [[0, Now(), module_id, dependency]]) + cursor.execute(*insert) class _Cursor(sqlite.Cursor): @@ -288,23 +337,10 @@ class Cursor(CursorInterface): return getattr(self.cursor, name) def execute(self, sql, params=None): - buf = "" - sql = re.sub(POSITION_PATTERN, r'POSITION(\1,', sql) - for nquote, quote in QUOTE_SEPARATION.findall(sql + "''"): - nquote = nquote.replace('?', '??') - nquote = nquote.replace('%s', '?') - nquote = nquote.replace('ilike', 'like') - nquote = re.sub(EXTRACT_PATTERN, r'EXTRACT("\1",', nquote) - buf += nquote + quote - sql = buf[:-2] - try: - if params: - res = self.cursor.execute(sql, params) - else: - res = self.cursor.execute(sql) - except Exception: - raise - return res + if params: + return self.cursor.execute(sql, params) + else: + return self.cursor.execute(sql) def close(self, close=False): self.cursor.close() @@ -319,21 +355,23 @@ class Cursor(CursorInterface): self._conn.rollback() def test(self): + sqlite_master = Table('sqlite_master') + select = sqlite_master.select(sqlite_master.name) + select.where = sqlite_master.type == 'table' + select.where &= sqlite_master.name.in_([ + 'ir_model', + 'ir_model_field', + 'ir_ui_view', + 'ir_ui_menu', + 'res_user', + 'res_group', + 'ir_module_module', + 'ir_module_module_dependency', + 'ir_translation', + 'ir_lang', + ]) try: - self.cursor.execute("SELECT name " - "FROM sqlite_master " - "WHERE type = 'table' AND name in (" - "'ir_model', " - "'ir_model_field', " - "'ir_ui_view', " - "'ir_ui_menu', " - "'res_user', " - "'res_group', " - "'ir_module_module', " - "'ir_module_module_dependency', " - "'ir_translation', " - "'ir_lang'" - ")") + self.cursor.execute(*select) except Exception: return False return len(self.cursor.fetchall()) != 0 @@ -348,15 +386,6 @@ class Cursor(CursorInterface): def has_constraint(self): return False - def limit_clause(self, select, limit=None, offset=None): - if limit is not None: - select += ' LIMIT %d' % limit - if offset is not None: - if limit is None: - select += ' LIMIT -1' - select += ' OFFSET %d' % offset - return select - sqlite.register_converter('NUMERIC', lambda val: Decimal(val)) if sys.version_info[0] == 2: sqlite.register_adapter(Decimal, lambda val: buffer(str(val))) diff --git a/trytond/backend/sqlite/fields.py b/trytond/backend/sqlite/fields.py deleted file mode 100644 index e9a325235e..0000000000 --- a/trytond/backend/sqlite/fields.py +++ /dev/null @@ -1,159 +0,0 @@ -#This file is part of Tryton. The COPYRIGHT file at the top level of -#this repository contains the full copyright notices and license terms. -from trytond.backend import fields - - -class Boolean(fields.Boolean): - - @staticmethod - def sql_format(value): - return value - - @staticmethod - def sql_type(field): - return ('BOOLEAN', 'BOOLEAN') - - -class Integer(fields.Integer): - - @staticmethod - def sql_format(value): - if value is None: - return value - return int(value) - - @staticmethod - def sql_type(field): - return ('INTEGER', 'INTEGER') - - -class BigInteger(fields.BigInteger, Integer): - pass - - -class Char(fields.Char): - - @staticmethod - def sql_type(field): - return ('VARCHAR', 'VARCHAR') - - -class Sha(fields.Sha): - - @staticmethod - def sql_type(field): - return ('VARCHAR', 'VARCHAR(40)') - - -class Text(fields.Text): - - @staticmethod - def sql_type(field): - return ('TEXT', 'TEXT') - - -class Float(fields.Float): - - @staticmethod - def sql_format(value): - if value is None: - return value - return float(value) - - @staticmethod - def sql_type(field): - return ('FLOAT', 'FLOAT') - - -class Numeric(fields.Numeric): - - @staticmethod - def sql_type(field): - return ('NUMERIC', 'NUMERIC') - - -class Date(fields.Date): - - @staticmethod - def sql_type(field): - return ('DATE', 'DATE') - - -class DateTime(fields.DateTime): - - @staticmethod - def sql_type(field): - return ('TIMESTAMP', 'TIMESTAMP') - - -class Timestamp(fields.Timestamp): - - @staticmethod - def sql_type(field): - return ('TIMESTAMP', 'TIMESTAMP') - - -class Time(fields.Time): - - @staticmethod - def sql_type(field): - return ('TIME', 'TIME') - - -class Binary(fields.Binary): - - @staticmethod - def sql_type(field): - return ('BLOB', 'BLOB') - - -class Selection(fields.Selection): - - @staticmethod - def sql_type(field): - return ('VARCHAR', 'VARCHAR') - - -class Reference(fields.Reference): - - @staticmethod - def sql_type(field): - return ('VARCHAR', 'VARCHAR') - - -class Many2One(fields.Many2One): - - @staticmethod - def sql_type(field): - return ('INTEGER', 'INTEGER') - - -class Dict(fields.Dict): - - @staticmethod - def sql_type(field): - return ('TEXT', 'TEXT') - -FIELDS = { - 'boolean': Boolean, - 'integer': Integer, - 'biginteger': BigInteger, - 'char': Char, - 'sha': Sha, - 'text': Text, - 'float': Float, - 'numeric': Numeric, - 'date': Date, - 'datetime': DateTime, - 'timestamp': Timestamp, - 'time': Time, - 'binary': Binary, - 'selection': Selection, - 'reference': Reference, - 'many2one': Many2One, - 'one2many': fields.One2Many, - 'many2many': fields.Many2Many, - 'function': fields.Function, - 'property': fields.Property, - 'dict': Dict, -} diff --git a/trytond/backend/sqlite/table.py b/trytond/backend/sqlite/table.py index d1e8559426..f2fafcb7b1 100644 --- a/trytond/backend/sqlite/table.py +++ b/trytond/backend/sqlite/table.py @@ -5,6 +5,8 @@ from trytond.backend.table import TableHandlerInterface import logging import re +__all__ = ['TableHandler'] + class TableHandler(TableHandlerInterface): def __init__(self, cursor, model, module_name=None, history=False): @@ -33,7 +35,7 @@ class TableHandler(TableHandlerInterface): @staticmethod def table_exist(cursor, table_name): cursor.execute("SELECT sql FROM sqlite_master " - "WHERE type = 'table' AND name = %s", + "WHERE type = 'table' AND name = ?", (table_name,)) res = cursor.fetchone() if not res: @@ -151,7 +153,7 @@ class TableHandler(TableHandlerInterface): self.cursor.execute('SELECT f.name, f.module ' 'FROM ir_model_field f ' 'JOIN ir_model m on (f.model=m.id) ' - 'WHERE m.model = %s', + 'WHERE m.model = ?', (self.object_name,)) for line in self.cursor.fetchall(): self._field2module[line[0]] = line[1] @@ -230,7 +232,7 @@ class TableHandler(TableHandlerInterface): if default_fun is not None: default = default_fun() self.cursor.execute('UPDATE "' + self.table_name + '" ' - 'SET "' + column_name + '" = %s', + 'SET "' + column_name + '" = ?', (column_format(default),)) self._update_definitions() diff --git a/trytond/cache.py b/trytond/cache.py index d34b124c03..2a528abe8d 100644 --- a/trytond/cache.py +++ b/trytond/cache.py @@ -4,7 +4,7 @@ import datetime from threading import Lock from trytond.transaction import Transaction from trytond.config import CONFIG -from trytond.backend import Database +from trytond import backend from trytond.tools import OrderedDict __all__ = ['Cache', 'LRUDict'] @@ -74,7 +74,7 @@ class Cache(object): def clean(dbname): if not CONFIG['multi_server']: return - database = Database(dbname).connect() + database = backend.get('Database')(dbname).connect() cursor = database.cursor() try: cursor.execute('SELECT "timestamp", "name" FROM ir_cache') @@ -104,7 +104,7 @@ class Cache(object): def resets(dbname): if not CONFIG['multi_server']: return - database = Database(dbname).connect() + database = backend.get('Database')(dbname).connect() cursor = database.cursor() try: with Cache._resets_lock: diff --git a/trytond/convert.py b/trytond/convert.py index 0545484c5d..d3e5e36517 100644 --- a/trytond/convert.py +++ b/trytond/convert.py @@ -8,12 +8,13 @@ import logging import traceback import sys import re +from sql import Table from itertools import izip from collections import defaultdict -from trytond.version import VERSION -from trytond.tools import safe_eval -from trytond.transaction import Transaction +from .version import VERSION +from .tools import safe_eval +from .transaction import Transaction CDATA_START = re.compile('^\s*\<\!\[cdata\[', re.IGNORECASE) CDATA_END = re.compile('\]\]\>\s*$', re.IGNORECASE) @@ -64,26 +65,37 @@ class MenuitemTagHandler: action_id = self.mh.get_id(values['action']) # TODO maybe use a prefetch for this: - cursor.execute(cursor.limit_clause( - "SELECT a.name, a.type, v.type, v.field_childs, icon.name " - "FROM ir_action a " - "LEFT JOIN ir_action_report report " - "ON (a.id = report.action) " - "LEFT JOIN ir_action_act_window act " - "ON (a.id = act.action) " - "LEFT JOIN ir_action_wizard wizard " - "ON (a.id = wizard.action) " - "LEFT JOIN ir_action_url url ON (a.id = url.action) " - "LEFT JOIN ir_action_act_window_view wv ON " - "(act.id = wv.act_window) " - "LEFT JOIN ir_ui_view v ON (v.id = wv.view) " - "LEFT JOIN ir_ui_icon icon ON (a.icon = icon.id) " - "WHERE report.id = %s " - "OR act.id = %s " - "OR wizard.id = %s " - "OR url.id = %s " - "ORDER by wv.sequence", 1), - (action_id, action_id, action_id, action_id)) + action = Table('ir_action') + report = Table('ir_action_report') + act_window = Table('ir_action_act_window') + wizard = Table('ir_action_wizard') + url = Table('ir_action_url') + act_window_view = Table('ir_action_act_window_view') + view = Table('ir_ui_view') + icon = Table('ir_ui_icon') + cursor.execute(*action.join( + report, 'LEFT', + condition=action.id == report.action + ).join(act_window, 'LEFT', + condition=action.id == act_window.action + ).join(wizard, 'LEFT', + condition=action.id == wizard.action + ).join(url, 'LEFT', + condition=action.id == url.action + ).join(act_window_view, 'LEFT', + condition=act_window.id == act_window_view.act_window + ).join(view, 'LEFT', + condition=view.id == act_window_view.view + ).join(icon, 'LEFT', + condition=action.icon == icon.id).select( + action.name, action.type, + view.type, view.field_childs, + icon.name, + where=(report.id == action_id) + | (act_window.id == action_id) + | (wizard.id == action_id) + | (url.id == action_id), + order_by=act_window_view.sequence, limit=1)) action_name, action_type, view_type, field_childs, icon_name = \ cursor.fetchone() diff --git a/trytond/error.py b/trytond/error.py index 0dc4a602dd..def502721d 100644 --- a/trytond/error.py +++ b/trytond/error.py @@ -7,6 +7,10 @@ from trytond.exceptions import UserError, UserWarning class WarningErrorMixin(object): + @classmethod + def _get_error_messages(cls): + return cls._error_messages.values() + @classmethod def raise_user_error(cls, error, error_args=None, error_description='', error_description_args=None, diff --git a/trytond/ir/action.py b/trytond/ir/action.py index b919a1fc0e..25a5bfa7c9 100644 --- a/trytond/ir/action.py +++ b/trytond/ir/action.py @@ -6,9 +6,12 @@ from operator import itemgetter from collections import defaultdict from functools import partial +from sql import Table +from sql.aggregate import Count + from ..model import ModelView, ModelStorage, ModelSQL, fields from ..tools import file_open, safe_eval -from ..backend import TableHandler +from .. import backend from ..pyson import PYSONEncoder, CONTEXT, PYSON from ..transaction import Transaction from ..pool import Pool @@ -124,6 +127,7 @@ class ActionKeyword(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(ActionKeyword, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) @@ -291,6 +295,7 @@ class ActionMixin(ModelSQL): def create(cls, vlist): pool = Pool() Action = pool.get('ir.action') + ir_action = cls.__table__() new_records = [] for values in vlist: later = {} @@ -313,8 +318,9 @@ class ActionMixin(ModelSQL): else: action = Action(values['action']) record, = super(ActionMixin, cls).create([values]) - cursor.execute('UPDATE "' + cls._table + '" SET id = %s ' - 'WHERE id = %s', (action.id, record.id)) + cursor.execute(*ir_action.update( + [ir_action.id], [action.id], + where=ir_action.id == record.id)) cursor.update_auto_increment(cls._table, action.id) record = cls(action.id) new_records.append(record) @@ -457,21 +463,24 @@ class ActionReport(ActionMixin, ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(ActionReport, cls).__register__(module_name) cursor = Transaction().cursor table = TableHandler(cursor, cls, module_name) + action_report = cls.__table__() # Migration from 1.0 report_name_uniq has been removed table.drop_constraint('report_name_uniq') # Migration from 1.0 output_format (m2o) is now extension (selection) if table.column_exist('output_format'): - cursor.execute( - 'SELECT report.id FROM "' + cls._table + '" report ' - 'JOIN ir_action_report_outputformat of ' - 'ON (report.output_format = of.id) ' - 'WHERE of.format = \'pdf\'') + outputformat = Table('ir_action_report_outputformat') + cursor.execute(*action_report.join( + outputformat, + condition=action_report.output_format == outputformat.id) + .select(action_report.id, + where=outputformat.format == 'pdf')) ids = [x[0] for x in cursor.fetchall()] with Transaction().set_user(0): @@ -481,34 +490,35 @@ class ActionReport(ActionMixin, ModelSQL, ModelView): table.drop_column("output_format") TableHandler.dropTable(cursor, 'ir.action.report.outputformat', - 'ir_action_report_outputformat') + 'ir_action_report_outputformat') # Migrate from 2.0 remove required on extension table.not_null_action('extension', action='remove') - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET extension = %s ' - 'WHERE extension = %s', ('', 'odt')) + cursor.execute(*action_report.update( + [action_report.extension], + [''], + where=action_report.extension == 'odt')) # Migration from 2.0 report_content_data renamed into # report_content_custom to remove base64 encoding if (table.column_exist('report_content_data') and table.column_exist('report_content_custom')): limit = cursor.IN_MAX - cursor.execute('SELECT COUNT(id) ' - 'FROM "' + cls._table + '"') + cursor.execute(*action_report.select( + Count(action_report.id))) report_count, = cursor.fetchone() for offset in range(0, report_count, limit): - cursor.execute(cursor.limit_clause( - 'SELECT id, report_content_data ' - 'FROM "' + cls._table + '"' - 'ORDER BY id', - limit, offset)) + cursor.execute(*action_report.select( + action_report.id, action_report.report_content_data, + order_by=action_report.id, + limit=limit, offset=offset)) for report_id, report in cursor.fetchall(): if report: report = buffer(base64.decodestring(str(report))) - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET report_content_custom = %s ' - 'WHERE id = %s', (report, report_id)) + cursor.execute(*action_report.update( + [action_report.report_content_custom], + [report], + where=action_report.id == report_id)) table.drop_column('report_content_data') @staticmethod @@ -691,13 +701,13 @@ class ActionActWindow(ActionMixin, ModelSQL, ModelView): @classmethod def __register__(cls, module_name): cursor = Transaction().cursor + act_window = cls.__table__() super(ActionActWindow, cls).__register__(module_name) # Migration from 2.0: new search_value format - cursor.execute('UPDATE "%s" ' - 'SET search_value = %%s ' - 'WHERE search_value = %%s' % cls._table, - ('[]', '{}')) + cursor.execute(*act_window.update( + [act_window.search_value], ['[]'], + where=act_window.search_value == '{}')) @staticmethod def default_type(): @@ -882,6 +892,7 @@ class ActionActWindowView(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(ActionActWindowView, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) diff --git a/trytond/ir/attachment.py b/trytond/ir/attachment.py index 7a31c9639d..09e340e817 100644 --- a/trytond/ir/attachment.py +++ b/trytond/ir/attachment.py @@ -6,9 +6,11 @@ try: except ImportError: hashlib = None import md5 +from sql.operators import Concat + from ..model import ModelView, ModelSQL, fields from ..config import CONFIG -from ..backend import TableHandler +from .. import backend from ..transaction import Transaction from ..pyson import Eval from ..pool import Pool @@ -64,19 +66,23 @@ class Attachment(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor super(Attachment, cls).__register__(module_name) table = TableHandler(cursor, cls, module_name) + attachment = cls.__table__() # Migration from 1.4 res_model and res_id merged into resource # Reference if table.column_exist('res_model') and \ table.column_exist('res_id'): table.drop_constraint('res_model_res_id_name') - cursor.execute('UPDATE "%s" ' - 'SET "resource" = "res_model"||\',\'||"res_id"' % cls._table) + cursor.execute(*attachment.update( + [attachment.resource], + [Concat(Concat(attachment.resource, ','), + attachment.res_id)])) table.drop_column('res_model') table.drop_column('res_id') @@ -137,6 +143,7 @@ class Attachment(ModelSQL, ModelView): if value is None: return cursor = Transaction().cursor + table = cls.__table__() db_name = cursor.dbname directory = os.path.join(CONFIG['data_path'], db_name) if not os.path.isdir(directory): @@ -154,11 +161,11 @@ class Attachment(ModelSQL, ModelView): with open(filename, 'rb') as file_p: data = file_p.read() if value != data: - cursor.execute('SELECT DISTINCT(collision) ' - 'FROM ir_attachment ' - 'WHERE digest = %s ' - 'AND collision != 0 ' - 'ORDER BY collision', (digest,)) + cursor.execute(*table.select(table.collision, + where=(table.digest == digest) + & (table.collision != 0), + group_by=table.collision, + order_by=table.collision)) collision2 = 0 for row in cursor.fetchall(): collision2 = row[0] diff --git a/trytond/ir/cron.py b/trytond/ir/cron.py index 48b2004b0f..3255b82718 100644 --- a/trytond/ir/cron.py +++ b/trytond/ir/cron.py @@ -12,7 +12,7 @@ from ..model import ModelView, ModelSQL, fields from ..tools import safe_eval, get_smtp_server from ..transaction import Transaction from ..pool import Pool -from ..backend import TableHandler +from .. import backend from ..config import CONFIG __all__ = [ @@ -69,7 +69,9 @@ class Cron(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor + cron = cls.__table__() # Migration from 2.0: rename numbercall, doall and nextcall table = TableHandler(cursor, cls, module_name) @@ -81,10 +83,9 @@ class Cron(ModelSQL, ModelView): super(Cron, cls).__register__(module_name) # Migration from 2.0: work_days removed - cursor.execute('UPDATE "%s" ' - 'SET interval_type = %%s ' - 'WHERE interval_type = %%s' % cls._table, - ('days', 'work_days')) + cursor.execute(*cron.update( + [cron.interval_type], ['days'], + where=cron.interval_type == 'work_days')) @staticmethod def default_next_call(): diff --git a/trytond/ir/model.py b/trytond/ir/model.py index ccf1b5e09b..0b5127ce9f 100644 --- a/trytond/ir/model.py +++ b/trytond/ir/model.py @@ -3,6 +3,8 @@ import datetime import re import heapq +from sql.aggregate import Max +from sql.conditionals import Case from collections import defaultdict from ..model import ModelView, ModelSQL, fields @@ -13,7 +15,7 @@ from ..cache import Cache from ..pool import Pool from ..pyson import Bool, Eval from ..rpc import RPC -from ..backend import TableHandler +from .. import backend try: from ..tools.StringMatcher import StringMatcher except ImportError: @@ -69,6 +71,39 @@ class Model(ModelSQL, ModelView): 'global_search': RPC(), }) + @classmethod + def register(cls, model, module_name): + pool = Pool() + Property = pool.get('ir.property') + cursor = Transaction().cursor + + ir_model = cls.__table__() + cursor.execute(*ir_model.select(ir_model.id, + where=ir_model.model == model.__name__)) + model_id = None + if cursor.rowcount == -1 or cursor.rowcount is None: + data = cursor.fetchone() + if data: + model_id, = data + elif cursor.rowcount != 0: + model_id, = cursor.fetchone() + if not model_id: + cursor.execute(*ir_model.insert( + [ir_model.model, ir_model.name, ir_model.info, + ir_model.module], + [[model.__name__, model._get_name(), model.__doc__, + module_name]])) + Property._models_get_cache.clear() + cursor.execute(*ir_model.select(ir_model.id, + where=ir_model.model == model.__name__)) + (model_id,) = cursor.fetchone() + elif model.__doc__: + cursor.execute(*ir_model.update( + [ir_model.name, ir_model.info], + [model._get_name(), model.__doc__], + where=ir_model.id == model_id)) + return model_id + @classmethod def validate(cls, models): super(Model, cls).validate(models) @@ -211,6 +246,65 @@ class ModelField(ModelSQL, ModelView): }) cls._order.insert(0, ('name', 'ASC')) + @classmethod + def register(cls, model, module_name, model_id): + pool = Pool() + Model = pool.get('ir.model') + cursor = Transaction().cursor + + ir_model_field = cls.__table__() + ir_model = Model.__table__() + + cursor.execute(*ir_model_field.join(ir_model, + condition=ir_model_field.model == ir_model.id + ).select(ir_model_field.id.as_('id'), + ir_model_field.name.as_('name'), + ir_model_field.field_description.as_('field_description'), + ir_model_field.ttype.as_('ttype'), + ir_model_field.relation.as_('relation'), + ir_model_field.module.as_('module'), + ir_model_field.help.as_('help'), + where=ir_model.model == model.__name__)) + model_fields = dict((f['name'], f) for f in cursor.dictfetchall()) + + for field_name, field in model._fields.iteritems(): + if hasattr(field, 'model_name'): + relation = field.model_name + elif hasattr(field, 'relation_name'): + relation = field.relation_name + else: + relation = None + + if field_name not in model_fields: + cursor.execute(*ir_model_field.insert( + [ir_model_field.model, ir_model_field.name, + ir_model_field.field_description, + ir_model_field.ttype, ir_model_field.relation, + ir_model_field.help, ir_model_field.module], + [[model_id, field_name, field.string, field._type, + relation, field.help, module_name]])) + elif (model_fields[field_name]['field_description'] != field.string + or model_fields[field_name]['ttype'] != field._type + or model_fields[field_name]['relation'] != relation + or model_fields[field_name]['help'] != field.help): + cursor.execute(*ir_model_field.update( + [ir_model_field.field_description, + ir_model_field.ttype, ir_model_field.relation, + ir_model_field.help], + [[field.string, field._type, relation, field.help]], + where=ir_model_field.id == + model_fields[field_name]['id'])) + + # Clean ir_model_field from field that are no more existing. + for field_name in model_fields: + if model_fields[field_name]['module'] == module_name \ + and field_name not in model._fields: + #XXX This delete field even when it is defined later + # in the module + cursor.execute(*ir_model_field.delete( + where=ir_model_field.id == + model_fields[field_name]['id'])) + @staticmethod def default_name(): return 'No Name' @@ -237,6 +331,7 @@ class ModelField(ModelSQL, ModelView): def read(cls, ids, fields_names=None): pool = Pool() Translation = pool.get('ir.translation') + Model = pool.get('ir.model') to_delete = [] if Transaction().context.get('language'): @@ -265,8 +360,9 @@ class ModelField(ModelSQL, ModelView): model_ids.add(rec['model']) model_ids = list(model_ids) cursor = Transaction().cursor - cursor.execute('SELECT id, model FROM ir_model WHERE id IN ' - '(' + ','.join(('%s',) * len(model_ids)) + ')', model_ids) + model = Model.__table__() + cursor.execute(*model.select(model.id, model.model, + where=model.id.in_(model_ids))) id2model = dict(cursor.fetchall()) trans_args = [] @@ -337,6 +433,7 @@ class ModelAccess(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor super(ModelAccess, cls).__register__(module_name) @@ -378,6 +475,9 @@ class ModelAccess(ModelSQL, ModelView): UserGroup = pool.get('res.user-res.group') cursor = Transaction().cursor user = Transaction().user + model_access = cls.__table__() + ir_model = Model.__table__() + user_group = UserGroup.__table__() access = {} for model in models: @@ -390,22 +490,19 @@ class ModelAccess(ModelSQL, ModelView): default = {'read': True, 'write': True, 'create': True, 'delete': True} access = dict((m, default) for m in models) - cursor.execute(('SELECT ' - 'm.model, ' - 'MAX(CASE WHEN a.perm_read THEN 1 ELSE 0 END), ' - 'MAX(CASE WHEN a.perm_write THEN 1 ELSE 0 END), ' - 'MAX(CASE WHEN a.perm_create THEN 1 ELSE 0 END), ' - 'MAX(CASE WHEN a.perm_delete THEN 1 ELSE 0 END) ' - 'FROM "%s" AS a ' - 'JOIN "%s" AS m ' - 'ON (a.model = m.id) ' - 'LEFT JOIN "%s" AS gu ' - 'ON (gu."group" = a."group") ' - 'WHERE m.model IN (' + ','.join(('%%s',) * len(models)) + ') ' - 'AND (gu."user" = %%s OR a."group" IS NULL) ' - 'GROUP BY m.model') - % (cls._table, Model._table, UserGroup._table), - list(models) + [Transaction().user]) + cursor.execute(*model_access.join(ir_model, 'LEFT', + condition=model_access.model == ir_model.id + ).join(user_group, 'LEFT', + condition=user_group.group == model_access.group + ).select( + ir_model.model, + Max(Case((model_access.perm_read, 1), else_=0)), + Max(Case((model_access.perm_write, 1), else_=0)), + Max(Case((model_access.perm_create, 1), else_=0)), + Max(Case((model_access.perm_delete, 1), else_=0)), + where=ir_model.model.in_(models) + & ((user_group.user == user) | (model_access.group == None)), + group_by=ir_model.model)) access.update(dict( (m, {'read': r, 'write': w, 'create': c, 'delete': d}) for m, r, w, c, d in cursor.fetchall())) @@ -476,6 +573,7 @@ class ModelFieldAccess(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor super(ModelFieldAccess, cls).__register__(module_name) @@ -516,8 +614,13 @@ class ModelFieldAccess(ModelSQL, ModelView): pool = Pool() Model = pool.get('ir.model') ModelField = pool.get('ir.model.field') + UserGroup = pool.get('res.user-res.group') cursor = Transaction().cursor user = Transaction().user + field_access = cls.__table__() + ir_model = Model.__table__() + model_field = ModelField.__table__() + user_group = UserGroup.__table__() accesses = {} for model in models: @@ -530,25 +633,22 @@ class ModelFieldAccess(ModelSQL, ModelView): default = {} accesses = dict((m, default) for m in models) - cursor.execute(('SELECT ' - 'm.model, ' - 'f.name, ' - 'MAX(CASE WHEN a.perm_read THEN 1 ELSE 0 END), ' - 'MAX(CASE WHEN a.perm_write THEN 1 ELSE 0 END), ' - 'MAX(CASE WHEN a.perm_create THEN 1 ELSE 0 END), ' - 'MAX(CASE WHEN a.perm_delete THEN 1 ELSE 0 END) ' - 'FROM "%s" AS a ' - 'JOIN "%s" AS f ' - 'ON (a.field = f.id) ' - 'JOIN "%s" AS m ' - 'ON (f.model = m.id) ' - 'LEFT JOIN "res_user-res_group" AS gu ' - 'ON (gu."group" = a."group") ' - 'WHERE m.model IN (' + ','.join(('%%s',) * len(models)) + ') ' - 'AND (gu."user" = %%s OR a."group" IS NULL) ' - 'GROUP BY m.model, f.name') - % (cls._table, ModelField._table, Model._table), - list(models) + [Transaction().user]) + cursor.execute(*field_access.join(model_field, + condition=field_access.field == model_field.id + ).join(ir_model, + condition=model_field.model == ir_model.id + ).join(user_group, 'LEFT', + condition=user_group.group == field_access.group + ).select( + ir_model.model, + model_field.name, + Max(Case((field_access.perm_read, 1), else_=0)), + Max(Case((field_access.perm_write, 1), else_=0)), + Max(Case((field_access.perm_create, 1), else_=0)), + Max(Case((field_access.perm_delete, 1), else_=0)), + where=ir_model.model.in_(models) + & ((user_group.user == user) | (field_access.group == None)), + group_by=[ir_model.model, model_field.name])) for m, f, r, w, c, d in cursor.fetchall(): accesses[m][f] = {'read': r, 'write': w, 'create': c, 'delete': d} for model, maccesses in accesses.iteritems(): @@ -690,7 +790,9 @@ class ModelData(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor + model_data = cls.__table__() super(ModelData, cls).__register__(module_name) @@ -698,8 +800,8 @@ class ModelData(ModelSQL, ModelView): # Migration from 2.6: remove inherit if table.column_exist('inherit'): - cursor.execute('DELETE FROM "' + cls._table + '" ' - 'WHERE inherit = %s', (True,)) + cursor.execute(*model_data.delete( + where=model_data.inherit == True)) table.drop_column('inherit', True) @staticmethod diff --git a/trytond/ir/module/module.py b/trytond/ir/module/module.py index 62d2e7ffef..5322ca49a1 100644 --- a/trytond/ir/module/module.py +++ b/trytond/ir/module/module.py @@ -4,7 +4,7 @@ from trytond.model import ModelView, ModelSQL, fields from trytond.modules import create_graph, get_module_list, get_module_info from trytond.wizard import Wizard, StateView, Button, StateTransition, \ StateAction -from trytond.backend import TableHandler +from trytond import backend from trytond.pool import Pool from trytond.transaction import Transaction from trytond.pyson import Eval @@ -339,6 +339,7 @@ class ModuleConfigWizardItem(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') table = TableHandler(Transaction().cursor, cls, module_name) # Migrate from 2.2 remove name diff --git a/trytond/ir/property.py b/trytond/ir/property.py index 4d82923ec3..9567c458b8 100644 --- a/trytond/ir/property.py +++ b/trytond/ir/property.py @@ -23,11 +23,15 @@ class Property(ModelSQL, ModelView): @classmethod def models_get(cls): + pool = Pool() + Model = pool.get('ir.model') models = cls._models_get_cache.get(None) if models: return models cursor = Transaction().cursor - cursor.execute('SELECT model, name FROM ir_model ORDER BY name ASC') + model = Model.__table__() + cursor.execute(*model.select(model.model, model.name, + order_by=model.name.asc)) models = cursor.fetchall() + [('', '')] cls._models_get_cache.set(None, models) return models diff --git a/trytond/ir/rule.py b/trytond/ir/rule.py index 496b1a59ec..74977d7abd 100644 --- a/trytond/ir/rule.py +++ b/trytond/ir/rule.py @@ -9,7 +9,7 @@ from ..tools import safe_eval from ..transaction import Transaction from ..cache import Cache from ..pool import Pool -from ..backend import TableHandler +from .. import backend __all__ = [ 'RuleGroup', 'Rule', @@ -111,6 +111,7 @@ class Rule(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(Rule, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) @@ -161,13 +162,13 @@ class Rule(ModelSQL, ModelView): # root user above constraint if Transaction().user == 0: if not Transaction().context.get('user'): - return '', [] + return with Transaction().set_user(Transaction().context['user']): return cls.domain_get(model_name, mode=mode) key = (model_name, mode) - domain = cls._domain_get_cache.get(key) - if domain: + domain = cls._domain_get_cache.get(key, False) + if domain is not False: return domain pool = Pool() @@ -178,32 +179,40 @@ class Rule(ModelSQL, ModelView): User_Group = pool.get('res.user-res.group') cursor = Transaction().cursor - cursor.execute('SELECT r.id FROM "' + cls._table + '" r ' - 'JOIN "' + RuleGroup._table + '" g ' - "ON (g.id = r.rule_group) " - 'JOIN "' + Model._table + '" m ON (g.model = m.id) ' - "WHERE m.model = %s " - "AND g.perm_" + mode + " " - "AND (g.id IN (" - 'SELECT rule_group ' - 'FROM "' + RuleGroup_User._table + '" ' - 'WHERE "user" = %s ' - "UNION SELECT rule_group " - 'FROM "' + RuleGroup_Group._table + '" g_rel ' - 'JOIN "' + User_Group._table + '" u_rel ' - 'ON (g_rel."group" = u_rel."group") ' - 'WHERE u_rel."user" = %s) ' - "OR default_p " - "OR g.global_p)", - (model_name, Transaction().user, Transaction().user)) + rule_table = cls.__table__() + rule_group = RuleGroup.__table__() + rule_group_user = RuleGroup_User.__table__() + rule_group_group = RuleGroup_Group.__table__() + user_group = User_Group.__table__() + model = Model.__table__() + user_id = Transaction().user + cursor.execute(*rule_table.join(rule_group, + condition=rule_group.id == rule_table.rule_group + ).join(model, + condition=rule_group.model == model.id + ).select(rule_table.id, + where=(model.model == model_name) + & getattr(rule_group, 'perm_%s' % mode) + & (rule_group.id.in_( + rule_group_user.select(rule_group_user.rule_group, + where=rule_group_user.user == user_id) + | rule_group_group.join( + user_group, + condition=(rule_group_group.group + == user_group.group) + ).select(rule_group_group.rule_group, + where=user_group.user == user_id) + ) + | rule_group.default_p + | rule_group.global_p + ))) ids = [x[0] for x in cursor.fetchall()] if not ids: - cls._domain_get_cache.set(key, ('', [])) - return '', [] + cls._domain_get_cache.set(key, None) + return obj = pool.get(model_name) clause = {} clause_global = {} - user_id = Transaction().user ctx = cls._get_context() # Use root user without context to prevent recursion with contextlib.nested(Transaction().set_user(0), @@ -220,20 +229,18 @@ class Rule(ModelSQL, ModelView): clause[rule.rule_group.id].append(dom) # Test if there is no rule_group that have no rule - cursor.execute('SELECT g.id FROM "' + RuleGroup._table + '" g ' - 'JOIN "' + Model._table + '" m ON (g.model = m.id) ' - 'WHERE m.model = %s ' - 'AND (g.id NOT IN (SELECT rule_group ' - 'FROM "' + cls._table + '")) ' - 'AND (g.id IN (SELECT rule_group ' - 'FROM "' + RuleGroup_User._table + '" ' - 'WHERE "user" = %s ' - 'UNION SELECT rule_group ' - 'FROM "' + RuleGroup_Group._table + '" g_rel ' - 'JOIN "' + User_Group._table + '" u_rel ' - 'ON g_rel."group" = u_rel."group" ' - 'WHERE u_rel."user" = %s))', - (model_name, user_id, user_id)) + cursor.execute(*rule_group.join(model, + condition=rule_group.model == model.id + ).select(rule_group.id, + where=(model.model == model_name) + & ~rule_group.id.in_(rule_table.select(rule_table.rule_group)) + & rule_group.id.in_(rule_group_user.select( + rule_group_user.rule_group, + where=rule_group_user.user == user_id) + | rule_group_group.join(user_group, + condition=rule_group_group.group == user_group.group + ).select(rule_group_group.rule_group, + where=user_group.user == user_id)))) fetchone = cursor.fetchone() if fetchone: group_id = fetchone[0] @@ -250,11 +257,10 @@ class Rule(ModelSQL, ModelView): # Use root to prevent infinite recursion with contextlib.nested(Transaction().set_user(0), Transaction().set_context(active_test=False, user=0)): - query, val = obj.search(clause, order=[], query_string=True) - query = '("%s".id IN (%s))' % (obj._table, query) + query = obj.search(clause, order=[], query=True) - cls._domain_get_cache.set(key, (query, val)) - return query, val + cls._domain_get_cache.set(key, query) + return query @classmethod def delete(cls, rules): diff --git a/trytond/ir/sequence.py b/trytond/ir/sequence.py index b0ba873c54..8637f483d0 100644 --- a/trytond/ir/sequence.py +++ b/trytond/ir/sequence.py @@ -3,13 +3,15 @@ from string import Template import time from itertools import izip +from sql import Flavor + from ..model import ModelView, ModelSQL, fields from ..tools import datetime_strftime from ..pyson import Eval, And from ..transaction import Transaction from ..pool import Pool from ..config import CONFIG -from ..backend import TableHandler +from .. import backend __all__ = [ 'SequenceType', 'Sequence', 'SequenceStrict', @@ -99,6 +101,7 @@ class Sequence(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor table = TableHandler(cursor, cls, module_name) @@ -242,17 +245,20 @@ class Sequence(ModelSQL, ModelView): def create_sql_sequence(self, number_next=None): 'Create the SQL sequence' cursor = Transaction().cursor + param = Flavor.get().param if self.type != 'incremental': return if number_next is None: number_next = self.number_next cursor.execute('CREATE SEQUENCE "' + self._sql_sequence_name - + '" INCREMENT BY %s START WITH %s', + + '" INCREMENT BY ' + param + ' START WITH ' + param, (self.number_increment, number_next)) def update_sql_sequence(self, number_next=None): 'Update the SQL sequence' + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor + param = Flavor.get().param exist = TableHandler.sequence_exist(cursor, self._sql_sequence_name) if self.type != 'incremental': if exist: @@ -264,7 +270,7 @@ class Sequence(ModelSQL, ModelView): if number_next is None: number_next = self.number_next cursor.execute('ALTER SEQUENCE "' + self._sql_sequence_name - + '" INCREMENT BY %s RESTART WITH %s', + + '" INCREMENT BY ' + param + ' RESTART WITH ' + param, (self.number_increment, number_next)) def delete_sql_sequence(self): diff --git a/trytond/ir/session.py b/trytond/ir/session.py index 1ebdd79654..e6512aea29 100644 --- a/trytond/ir/session.py +++ b/trytond/ir/session.py @@ -9,7 +9,7 @@ import datetime from trytond.model import ModelSQL, fields from trytond.config import CONFIG -from ..backend import TableHandler +from .. import backend from ..transaction import Transaction __all__ = [ @@ -31,6 +31,7 @@ class Session(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(Session, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) diff --git a/trytond/ir/translation.py b/trytond/ir/translation.py index a754bb62af..fb88e10a3d 100644 --- a/trytond/ir/translation.py +++ b/trytond/ir/translation.py @@ -17,12 +17,17 @@ except ImportError: from functools import reduce from lxml import etree from itertools import izip +from sql import Column +from sql.functions import Substring, Position +from sql.conditionals import Case +from sql.operators import Or, And +from sql.aggregate import Max from ..model import ModelView, ModelSQL, fields from ..wizard import Wizard, StateView, StateTransition, StateAction, \ Button from ..tools import file_open, reduce_ids -from ..backend import TableHandler, FIELDS +from .. import backend from ..pyson import PYSONEncoder from ..transaction import Transaction from ..pool import Pool @@ -85,14 +90,16 @@ class Translation(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor + ir_translation = cls.__table__() table = TableHandler(cursor, cls, module_name) # Migration from 1.8: new field src_md5 src_md5_exist = table.column_exist('src_md5') if not src_md5_exist: table.add_raw_column('src_md5', - FIELDS[cls.src_md5._type].sql_type(cls.src_md5), - FIELDS[cls.src_md5._type].sql_format, None, + cls.src_md5.sql_type(), + cls.src_md5.sql_format, None, cls.src_md5.size, string=cls.src_md5.string) table.drop_constraint('translation_uniq') table.index_action(['lang', 'type', 'name', 'src'], 'remove') @@ -116,14 +123,195 @@ class Translation(ModelSQL, ModelView): table.not_null_action('src_md5', action='add') # Migration from 2.2 - cursor.execute("UPDATE " + cls._table + " " - "SET res_id = %s " - "WHERE res_id = %s", - (None, 0)) + cursor.execute(*ir_translation.update([ir_translation.res_id], + [None], where=ir_translation.res_id == 0)) table = TableHandler(Transaction().cursor, cls, module_name) table.index_action(['lang', 'type', 'name'], 'add') + @classmethod + def register_model(cls, model, module_name): + cursor = Transaction().cursor + ir_translation = cls.__table__() + + if not model.__doc__: + return + + name = model.__name__ + ',name' + src = model._get_name() + cursor.execute(*ir_translation.select(ir_translation.id, + where=(ir_translation.lang == 'en_US') + & (ir_translation.type == 'model') + & (ir_translation.name == name) + & ((ir_translation.res_id == None) + | (ir_translation.res_id == 0)))) + trans_id = None + if cursor.rowcount == -1 or cursor.rowcount is None: + data = cursor.fetchone() + if data: + trans_id, = data + elif cursor.rowcount != 0: + trans_id, = cursor.fetchone() + src_md5 = Translation.get_src_md5(src) + if trans_id is None: + cursor.execute(*ir_translation.insert( + [Column(ir_translation, c) + for c in ('name', 'lang', 'type', 'src', 'src_md5', + 'value', 'module', 'fuzzy')], + [[name, 'en_US', 'model', src, src_md5, '', + module_name, False]])) + else: + cursor.execute(*ir_translation.update( + [ir_translation.src, ir_translation.src_md5], + [src, src_md5], + where=ir_translation.id == trans_id)) + + @classmethod + def register_fields(cls, model, module_name): + cursor = Transaction().cursor + ir_translation = cls.__table__() + + # Prefetch field translations + trans_fields = {} + trans_help = {} + trans_selection = {} + if model._fields: + names = ['%s,%s' % (model.__name__, f) for f in model._fields] + cursor.execute(*ir_translation.select(ir_translation.id, + ir_translation.name, ir_translation.src, + ir_translation.type, + where=((ir_translation.lang == 'en_US') + & ir_translation.type.in_( + ('field', 'help', 'selection')) + & ir_translation.name.in_(names)))) + for trans in cursor.dictfetchall(): + if trans['type'] == 'field': + trans_fields[trans['name']] = trans + elif trans['type'] == 'help': + trans_help[trans['name']] = trans + elif trans['type'] == 'selection': + trans_selection.setdefault(trans['name'], {}) + trans_selection[trans['name']][trans['src']] = trans + + def update_insert_field(field, trans_name): + string_md5 = cls.get_src_md5(field.string) + if trans_name not in trans_fields: + cursor.execute(*ir_translation.insert( + [ir_translation.name, ir_translation.lang, + ir_translation.type, ir_translation.src, + ir_translation.src_md5, ir_translation.value, + ir_translation.module, ir_translation.fuzzy], + [[trans_name, 'en_US', 'field', field.string, + string_md5, '', module_name, False]])) + elif trans_fields[trans_name]['src'] != field.string: + cursor.execute(*ir_translation.update( + [ir_translation.src, ir_translation.src_md5], + [field.string, string_md5], + where=ir_translation.id == + trans_fields[trans_name]['id'])) + + def update_insert_help(field, trans_name): + help_md5 = cls.get_src_md5(field.help) + if trans_name not in trans_help: + if field.help: + cursor.execute(*ir_translation.insert( + [ir_translation.name, ir_translation.lang, + ir_translation.type, ir_translation.src, + ir_translation.src_md5, ir_translation.value, + ir_translation.module, ir_translation.fuzzy], + [[trans_name, 'en_US', 'help', field.help, + help_md5, '', module_name, False]])) + elif trans_help[trans_name]['src'] != field.help: + cursor.execute(*ir_translation.update( + [ir_translation.src, ir_translation.src_md5], + [field.help, help_md5], + where=ir_translation.id == + trans_help[trans_name]['id'])) + + def insert_selection(field, trans_name): + for (_, val) in field.selection: + if (trans_name not in trans_selection + or val not in trans_selection[trans_name]): + val_md5 = cls.get_src_md5(val) + cursor.execute(*ir_translation.insert( + [ir_translation.name, ir_translation.lang, + ir_translation.type, ir_translation.src, + ir_translation.src_md5, ir_translation.value, + ir_translation.module, ir_translation.fuzzy], + [[trans_name, 'en_US', 'selection', val, val_md5, + '', module_name, False]])) + + for field_name, field in model._fields.iteritems(): + trans_name = model.__name__ + ',' + field_name + update_insert_field(field, trans_name) + update_insert_help(field, trans_name) + if (hasattr(field, 'selection') + and isinstance(field.selection, (tuple, list)) + and getattr(field, 'translate_selection', True)): + insert_selection(field, trans_name) + + @classmethod + def register_error_messages(cls, model, module_name): + cursor = Transaction().cursor + ir_translation = cls.__table__() + + cursor.execute(*ir_translation.select( + ir_translation.id, ir_translation.src, + where=((ir_translation.lang == 'en_US') + & (ir_translation.type == 'error') + & (ir_translation.name == model.__name__)))) + trans_error = dict((t['src'], t) for t in cursor.dictfetchall()) + + errors = model._get_error_messages() + for error in set(errors): + if error not in trans_error: + error_md5 = Translation.get_src_md5(error) + cursor.execute(*ir_translation.insert( + [ir_translation.name, ir_translation.lang, + ir_translation.type, ir_translation.src, + ir_translation.src_md5, ir_translation.value, + ir_translation.module, ir_translation.fuzzy], + [[model.__name__, 'en_US', 'error', error, error_md5, + '', module_name, False]])) + + @classmethod + def register_wizard(cls, wizard, module_name): + cursor = Transaction().cursor + ir_translation = cls.__table__() + + # Prefetch button translations + cursor.execute(*ir_translation.select( + ir_translation.id, ir_translation.name, ir_translation.src, + where=((ir_translation.lang == 'en_US') + & (ir_translation.type == 'wizard_button') + & (ir_translation.name.like(wizard.__name__ + ',%'))))) + trans_buttons = dict((t['name'], t) for t in cursor.dictfetchall()) + + def update_insert_button(state_name, button): + trans_name = '%s,%s,%s' % ( + wizard.__name__, state_name, button.state) + src_md5 = cls.get_src_md5(button.string) + if trans_name not in trans_buttons: + cursor.execute(*ir_translation.insert( + [ir_translation.name, ir_translation.lang, + ir_translation.type, ir_translation.src, + ir_translation.src_md5, ir_translation.value, + ir_translation.module, ir_translation.fuzzy], + [[trans_name, 'en_US', 'wizard_button', button.string, + src_md5, '', module_name, False]])) + elif trans_buttons[trans_name] != button.string: + cursor.execute(*ir_translation.update( + [ir_translation.src, ir_translation.src_md5], + [button.string, src_md5], + where=ir_translation.id == + trans_buttons[trans_name]['id'])) + + for state_name, state in wizard.states.iteritems(): + if not isinstance(state, StateView): + continue + for button in state.buttons: + update_insert_button(state_name, button) + @staticmethod def default_fuzzy(): return False @@ -144,14 +332,13 @@ class Translation(ModelSQL, ModelView): @classmethod def search_model(cls, name, clause): - cursor = Transaction().cursor - cursor.execute('SELECT id FROM "%s" ' - 'WHERE SUBSTR(name, 1, ' - 'CASE WHEN POSITION(\',\' IN name) > 0 ' - 'THEN POSITION(\',\' IN name) - 1 ' - 'ELSE 0 END) %s %%s' % - (cls._table, clause[1]), (clause[2],)) - return [('id', 'in', [x[0] for x in cursor.fetchall()])] + table = cls.__table__() + _, operator, value = clause + Operator = fields.SQL_OPERATORS[operator] + return Operator(Substring(table.name, 1, + Case((Position(',', table.name) > 0, + Position(',', table.name) - 1), + else_=0)), value) @classmethod def get_language(cls): @@ -223,24 +410,25 @@ class Translation(ModelSQL, ModelView): to_fetch = ids if to_fetch: cursor = Transaction().cursor - fuzzy_sql = 'AND fuzzy = %s ' - fuzzy = [False] + table = cls.__table__() + fuzzy_sql = table.fuzzy == False if Transaction().context.get('fuzzy_translation', False): - fuzzy_sql = '' - fuzzy = [] - for i in range(0, len(to_fetch), cursor.IN_MAX): - sub_to_fetch = to_fetch[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('res_id', sub_to_fetch) - cursor.execute('SELECT res_id, value ' - 'FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND value != \'\' ' - 'AND value IS NOT NULL ' - + fuzzy_sql + - 'AND ' + red_sql, - [lang, ttype, name] + fuzzy + red_ids) + fuzzy_sql = None + in_max = cursor.IN_MAX / 7 + for i in range(0, len(to_fetch), in_max): + sub_to_fetch = to_fetch[i:i + in_max] + red_sql = reduce_ids(table.res_id, sub_to_fetch) + where = And(((table.lang == lang), + (table.type == ttype), + (table.name == name), + (table.value != ''), + (table.value != None), + red_sql, + )) + if fuzzy_sql: + where &= fuzzy_sql + cursor.execute(*table.select(table.res_id, table.value, + where=where)) for res_id, value in cursor.fetchall(): # Don't store fuzzy translation in cache if not Transaction().context.get( @@ -406,29 +594,17 @@ class Translation(ModelSQL, ModelView): return trans cursor = Transaction().cursor + table = cls.__table__() + where = ((table.lang == lang) + & (table.type == ttype) + & (table.name == name) + & (table.value != '') + & (table.value != None) + & (table.fuzzy == False) + & (table.res_id == None)) if source is not None: - cursor.execute('SELECT value ' - 'FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND src = %s ' - 'AND value != \'\' ' - 'AND value IS NOT NULL ' - 'AND fuzzy = %s ' - 'AND res_id IS NULL', - (lang, ttype, name, source, False)) - else: - cursor.execute('SELECT value ' - 'FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND value != \'\' ' - 'AND value IS NOT NULL ' - 'AND fuzzy = %s ' - 'AND res_id IS NULL', - (lang, ttype, name, False)) + where &= table.src == source + cursor.execute(*table.select(table.value, where=where)) res = cursor.fetchone() if res: cls._translation_cache.set((lang, ttype, name, source), res[0]) @@ -447,6 +623,7 @@ class Translation(ModelSQL, ModelView): res = {} clause = [] cursor = Transaction().cursor + table = cls.__table__() if len(args) > cursor.IN_MAX: for i in range(0, len(args), cursor.IN_MAX): sub_args = args[i:i + cursor.IN_MAX] @@ -464,32 +641,24 @@ class Translation(ModelSQL, ModelView): else: res[(name, ttype, lang, source)] = None cls._translation_cache.set((lang, ttype, name, source), None) + where = And(((table.lang == lang), + (table.type == ttype), + (table.name == name), + (table.value != ''), + (table.value != None), + (table.fuzzy == False), + (table.res_id == None), + )) if source is not None: - clause += [('(lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND src = %s ' - 'AND value != \'\' ' - 'AND value IS NOT NULL ' - 'AND fuzzy = %s ' - 'AND res_id IS NULL)', - (lang, ttype, name, source, False))] - else: - clause += [('(lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND value != \'\' ' - 'AND value IS NOT NULL ' - 'AND fuzzy = %s ' - 'AND res_id IS NULL)', - (lang, ttype, name, False))] + where &= table.src == source + clause.append(where) if clause: - for i in range(0, len(clause), cursor.IN_MAX): - sub_clause = clause[i:i + cursor.IN_MAX] - cursor.execute('SELECT lang, type, name, src, value ' - 'FROM ir_translation ' - 'WHERE ' + ' OR '.join(x[0] for x in sub_clause), - reduce(lambda x, y: x + y, [x[1] for x in sub_clause])) + in_max = cursor.IN_MAX / 7 + for i in range(0, len(clause), in_max): + cursor.execute(*table.select( + table.lang, table.type, table.name, table.src, + table.value, + where=Or(clause[i:i + in_max]))) for lang, ttype, name, source, value in cursor.fetchall(): if (name, ttype, lang, source) not in args: source = None @@ -511,32 +680,28 @@ class Translation(ModelSQL, ModelView): vlist = [x.copy() for x in vlist] cursor = Transaction().cursor + table = cls.__table__() for vals in vlist: if not vals.get('module'): if Transaction().context.get('module'): vals['module'] = Transaction().context['module'] elif vals.get('type', '') in ('odt', 'view', 'wizard_button', 'selection', 'error'): - cursor.execute('SELECT module FROM ir_translation ' - 'WHERE name = %s ' - 'AND res_id = %s ' - 'AND lang = %s ' - 'AND type = %s ' - 'AND src = %s ', - (vals.get('name') or '', vals.get('res_id') or 0, - 'en_US', vals.get('type') or '', vals.get('src') - or '')) + cursor.execute(*table.select(table.module, + where=(table.name == vals.get('name') or '') + & (table.res_id == vals.get('res_id') or 0) + & (table.lang == 'en_US') + & (table.type == vals.get('type') or '') + & (table.src == vals.get('src') or ''))) fetchone = cursor.fetchone() if fetchone: vals['module'] = fetchone[0] else: - cursor.execute('SELECT module, src FROM ir_translation ' - 'WHERE name = %s ' - 'AND res_id = %s ' - 'AND lang = %s ' - 'AND type = %s', - (vals.get('name') or '', vals.get('res_id') or 0, - 'en_US', vals.get('type') or '')) + cursor.execute(*table.select(table.module, table.src, + where=(table.name == vals.get('name') or '') + & (table.res_id == vals.get('res_id') or 0) + & (table.lang == 'en_US') + & (table.type == vals.get('type') or ''))) fetchone = cursor.fetchone() if fetchone: vals['module'], vals['src'] = fetchone @@ -778,13 +943,14 @@ class TranslationSet(Wizard): return cursor = Transaction().cursor + translation = Transaction.__table__() for report in reports: - cursor.execute('SELECT id, name, src FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND module = %s', - ('en_US', 'odt', report.report_name, report.module or '')) + cursor.extend(*translation.select( + translation.id, translation.name, translation.src, + where=(translation.lang == 'en_US') + & (translation.type == 'odt') + & (translation.name == report.report_name) + & (translation.module == report.module or ''))) trans_reports = {} for trans in cursor.dictfetchall(): trans_reports[trans['src']] = trans @@ -839,35 +1005,31 @@ class TranslationSet(Wizard): done = True break if seqmatch.ratio() > 0.6: - cursor.execute('UPDATE ir_translation ' - 'SET src = %s, ' - 'fuzzy = %s, ' - 'src_md5 = %s ' - 'WHERE name = %s ' - 'AND type = %s ' - 'AND src = %s ' - 'AND module = %s', - (string, True, src_md5, report.report_name, - 'odt', string_trans, report.module)) + cursor.execute(*translation.update( + [translation.src, translation.fuzzy, + translation.src_md5], + [string, True, src_md5], + where=(translation.name == report.report_name) + & (translation.type == 'odt') + & (translation.src == string_trans) + & (translation.module == report.module))) del trans_reports[string_trans] done = True break if not done: - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, value, module, fuzzy, ' - 'src_md5)' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (report.report_name, 'en_US', 'odt', string, '', - report.module, False, src_md5)) + cursor.execute(*translation.insert( + [translation.name, translation.lang, + translation.type, translation.src, + translation.value, translation.module, + translation.fuzzy, translation.src_md5], + [[report.report_name, 'en_US', 'odt', string, '', + report.module, False, src_md5]])) if strings: - cursor.execute('DELETE FROM ir_translation ' - 'WHERE name = %s ' - 'AND type = %s ' - 'AND module = %s ' - 'AND src NOT IN ' - '(' + ','.join(('%s',) * len(strings)) + ')', - (report.report_name, 'odt', report.module) + - tuple(strings)) + cursor.execute(*translation.delete( + where=(translation.name == report.report_name) + & (translation.type == 'odt') + & (translation.module == report.module) + & ~translation.src.in_(strings))) def _translate_view(self, element): strings = [] @@ -883,6 +1045,7 @@ class TranslationSet(Wizard): def set_view(self): pool = Pool() View = pool.get('ir.ui.view') + Translation = pool.get('ir.translation') with Transaction().set_context(active_test=False): views = View.search([]) @@ -890,14 +1053,14 @@ class TranslationSet(Wizard): if not views: return cursor = Transaction().cursor - + translation = Translation.__table__ for view in views: - cursor.execute('SELECT id, name, src FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND module = %s', - ('en_US', 'view', view.model, view.module)) + cursor.execute(*translation.select( + translation.id, translation.name, translation.src, + where=(translation.lang == 'en_US') + & (translation.type == 'view') + & (translation.name == view.model) + & (translation.module == view.module))) trans_views = {} for trans in cursor.dictfetchall(): trans_views[trans['src']] = trans @@ -939,31 +1102,29 @@ class TranslationSet(Wizard): done = True break if seqmatch.ratio() > 0.6: - cursor.execute('UPDATE ir_translation ' - 'SET src = %s, ' - 'src_md5 = %s, ' - 'fuzzy = %s ' - 'WHERE id = %s ', - (string, string_md5, True, - trans_views[string_trans]['id'])) + cursor.execute(*translation.update( + [translation.src, translation.src_md5, + translation.fuzzy], + [string, string_md5, True], + where=(translation.id == + trans_views[string_trans]['id']))) del trans_views[string_trans] done = True break if not done: - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, ' - 'fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (view.model, 'en_US', 'view', string, string_md5, '', - view.module, False)) + cursor.execute(*translation.insert( + [translation.name, translation.lang, + translation.type, translation.src, + translation.src_md5, translation.value, + translation.module, translation.fuzzy], + [[view.model, 'en_US', 'view', string, string_md5, + '', view.module, False]])) if strings: - cursor.execute('DELETE FROM ir_translation ' - 'WHERE name = %s ' - 'AND type = %s ' - 'AND module = %s ' - 'AND src NOT IN ' - '(' + ','.join(('%s',) * len(strings)) + ')', - (view.model, 'view', view.module) + tuple(strings)) + cursor.execute(*translation.delete( + where=(translation.name == view.model) + & (translation.type == 'view') + & (translation.module == view.module) + & ~translation.src.in_(strings))) def transition_set_(self): self.set_report() @@ -1220,18 +1381,17 @@ class TranslationUpdate(Wizard): pool = Pool() Translation = pool.get('ir.translation') cursor = Transaction().cursor + translation = Translation.__table__() lang = self.start.language.code - cursor.execute('SELECT name, res_id, type, src, module ' - 'FROM ir_translation ' - 'WHERE lang=\'en_US\' ' - 'AND type in (\'odt\', \'view\', \'wizard_button\', ' - ' \'selection\', \'error\') ' - 'EXCEPT SELECT name, res_id, type, src, module ' - 'FROM ir_translation ' - 'WHERE lang=%s ' - 'AND type in (\'odt\', \'view\', \'wizard_button\', ' - ' \'selection\', \'error\')', - (lang,)) + types = ['odt', 'view', 'wizard_button', 'selection', 'error'] + columns = [translation.name, translation.res_id, translation.type, + translation.src, translation.module] + cursor.execute(*(translation.select(*columns, + where=(translation.lang == 'en_US') + & translation.type.in_(types)) + - translation.select(*columns, + where=(translation.lang == lang) + & translation.type.in_(types)))) to_create = [] for row in cursor.dictfetchall(): to_create.append({ @@ -1245,15 +1405,15 @@ class TranslationUpdate(Wizard): if to_create: with Transaction().set_user(0): Translation.create(to_create) - cursor.execute('SELECT name, res_id, type, module ' - 'FROM ir_translation ' - 'WHERE lang=\'en_US\' ' - 'AND type in (\'field\', \'model\', \'help\') ' - 'EXCEPT SELECT name, res_id, type, module ' - 'FROM ir_translation ' - 'WHERE lang=%s ' - 'AND type in (\'field\', \'model\', \'help\')', - (lang,)) + types = ['field', 'model', 'help'] + columns = [translation.name, translation.res_id, translation.type, + translation.module] + cursor.execute(*(translation.select(*columns, + where=(translation.lang == 'en_US') + & translation.type.in_(types)) + - translation.select(*columns, + where=(translation.lang == lang) + & translation.type.in_(types)))) to_create = [] for row in cursor.dictfetchall(): to_create.append({ @@ -1266,51 +1426,49 @@ class TranslationUpdate(Wizard): if to_create: with Transaction().set_user(0): Translation.create(to_create) - cursor.execute('SELECT name, res_id, type, src ' - 'FROM ir_translation ' - 'WHERE lang=\'en_US\' ' - 'AND type in (\'field\', \'model\', \'selection\', ' - '\'help\') ' - 'EXCEPT SELECT name, res_id, type, src ' - 'FROM ir_translation ' - 'WHERE lang=%s ' - 'AND type in (\'field\', \'model\', \'selection\', ' - '\'help\')', - (lang,)) + types = ['field', 'model', 'selection', 'help'] + columns = [translation.name, translation.res_id, translation.type, + translation.src] + cursor.execute(*(translation.select(*columns, + where=(translation.lang == 'en_US') + & translation.type.in_(types)) + - translation.select(*columns, + where=(translation.lang == lang) + & translation.type.in_(types)))) for row in cursor.dictfetchall(): - cursor.execute('UPDATE ir_translation ' - 'SET fuzzy = %s, ' - 'src = %s ' - 'WHERE name = %s ' - 'AND type = %s ' - 'AND lang = %s ' - + ('AND res_id = %s' if row['res_id'] - else 'AND res_id is NULL'), - (True, row['src'], row['name'], row['type'], lang) - + ((row['res_id'],) if row['res_id'] else ())) + cursor.execute(*translation.update( + [translation.fuzzy, translation.src], + [True, row['src']], + where=(translation.name == row['name']) + & (translation.type == row['type']) + & (translation.lang == lang) + & (translation.res_id == (row['res_id'] or None)))) - cursor.execute('SELECT src, MAX(value) AS value FROM ir_translation ' - 'WHERE lang = %s ' - 'AND src IN (' - 'SELECT src FROM ir_translation ' - 'WHERE (value = \'\' OR value IS NULL) ' - 'AND lang = %s) ' - 'AND value != \'\' AND value IS NOT NULL ' - 'GROUP BY src', (lang, lang)) + cursor.execute(*translation.select( + translation.src, Max(translation.value).as_('value'), + where=(translation.lang == lang) + & translation.src.in_( + translation.select(translation.src, + where=((translation.value == '') + | (translation.value == None)) + & (translation.lang == lang))) + & (translation.value != '') + & (translation.value != None), + group_by=translation.src)) for row in cursor.dictfetchall(): - cursor.execute('UPDATE ir_translation ' - 'SET fuzzy = %s, ' - 'value = %s ' - 'WHERE src = %s ' - 'AND (value = \'\' OR value IS NULL) ' - 'AND lang = %s', - (True, row['value'], row['src'], lang)) + cursor.execute(*translation.update( + [translation.fuzzy, translation.value], + [True, row['value']], + where=(translation.src == row['src']) + & ((translation.value == '') | (translation.value == None)) + & (translation.lang == lang))) - cursor.execute('UPDATE ir_translation ' - 'SET fuzzy = %s ' - 'WHERE (value = \'\' OR value IS NULL) ' - 'AND lang = %s', (False, lang,)) + cursor.execute(*translation.update( + [translation.fuzzy], + [False], + where=((translation.value == '') | (translation.value == None)) + & (translation.lang == lang))) action['pyson_domain'] = PYSONEncoder().encode([ ('module', '!=', False), diff --git a/trytond/ir/trigger.py b/trytond/ir/trigger.py index 7ee4356ece..79fe2ca16e 100644 --- a/trytond/ir/trigger.py +++ b/trytond/ir/trigger.py @@ -2,10 +2,13 @@ #this repository contains the full copyright notices and license terms. import datetime import time +from sql import Literal +from sql.aggregate import Count, Max + from ..model import ModelView, ModelSQL, fields from ..pyson import Eval from ..tools import safe_eval -from ..backend import TableHandler +from .. import backend from ..tools import reduce_ids from ..transaction import Transaction from ..cache import Cache @@ -169,19 +172,20 @@ class Trigger(ModelSQL, ModelView): Model = pool.get(trigger.model.model) ActionModel = pool.get(trigger.action_model.model) cursor = Transaction().cursor + in_max = cursor.IN_MAX + trigger_log = TriggerLog.__table__() ids = map(int, records) # Filter on limit_number if trigger.limit_number: new_ids = [] - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('"record_id"', sub_ids) - cursor.execute('SELECT "record_id", COUNT(1) FROM "%s" ' - 'WHERE %s AND "trigger" = %%s ' - 'GROUP BY "record_id"' - % (TriggerLog._table, red_sql), - red_ids + [trigger.id]) + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + red_sql = reduce_ids(trigger_log.record_id, sub_ids) + cursor.execute(*trigger_log.select( + trigger_log.record_id, Count(Literal(1)), + where=red_sql & (trigger_log.trigger == trigger.id), + group_by=trigger_log.record_id)) number = dict(cursor.fetchall()) for record_id in sub_ids: if record_id not in number: @@ -194,15 +198,13 @@ class Trigger(ModelSQL, ModelView): # Filter on minimum_delay if trigger.minimum_delay: new_ids = [] - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('"record_id"', sub_ids) - cursor.execute('SELECT "record_id", MAX("create_date") ' - 'FROM "%s" ' - 'WHERE %s AND "trigger" = %%s ' - 'GROUP BY "record_id"' - % (TriggerLog._table, red_sql), - red_ids + [trigger.id]) + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + red_sql = reduce_ids(trigger_log.record_id, sub_ids) + cursor.execute(*trigger_log.select( + trigger_log.record_id, Max(trigger_log.create_date), + where=red_sql & (trigger_log.trigger == trigger.id), + group_by=trigger_log.record_id)) delay = dict(cursor.fetchall()) for record_id in sub_ids: if record_id not in delay: @@ -289,6 +291,7 @@ class TriggerLog(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(TriggerLog, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) diff --git a/trytond/ir/ui/menu.py b/trytond/ir/ui/menu.py index c3356f6f8c..edb7039d71 100644 --- a/trytond/ir/ui/menu.py +++ b/trytond/ir/ui/menu.py @@ -78,8 +78,8 @@ class UIMenu(ModelSQL, ModelView): ondelete='CASCADE') groups = fields.Many2Many('ir.ui.menu-res.group', 'menu', 'group', 'Groups') - complete_name = fields.Function(fields.Char('Complete Name', - order_field='name'), 'get_rec_name', searcher='search_rec_name') + complete_name = fields.Function(fields.Char('Complete Name'), + 'get_rec_name', searcher='search_rec_name') icon = fields.Selection('list_icons', 'Icon', translate=False) action = fields.Function(fields.Reference('Action', selection=[ @@ -101,6 +101,10 @@ class UIMenu(ModelSQL, ModelView): 'not allowed to contain "%s".' % SEPARATOR), }) + @classmethod + def order_complet_name(cls, tables): + return cls.name.convert_order('name', tables, cls) + @staticmethod def default_icon(): return 'tryton-open' @@ -163,10 +167,10 @@ class UIMenu(ModelSQL, ModelView): @classmethod def search(cls, domain, offset=0, limit=None, order=None, count=False, - query_string=False): + query=False): menus = super(UIMenu, cls).search(domain, offset=offset, limit=limit, - order=order, count=False, query_string=query_string) - if query_string: + order=order, count=False, query=query) + if query: return menus if menus: @@ -262,9 +266,7 @@ class UIMenuFavorite(ModelSQL, ModelView): menu = fields.Many2One('ir.ui.menu', 'Menu', required=True, ondelete='CASCADE') - sequence = fields.Integer('Sequence', - order_field='(%(table)s.sequence IS NOT NULL) %(order)s, ' - '%(table)s.sequence %(order)s') + sequence = fields.Integer('Sequence') user = fields.Many2One('res.user', 'User', required=True, ondelete='CASCADE') @@ -281,6 +283,11 @@ class UIMenuFavorite(ModelSQL, ModelView): ('id', 'DESC'), ] + @staticmethod + def order_sequence(tables): + table, _ = tables[None] + return [table.sequence == None, table.sequence] + @staticmethod def default_user(): return Transaction().user diff --git a/trytond/ir/ui/menu.xml b/trytond/ir/ui/menu.xml index 423b4bf599..4a0a610471 100644 --- a/trytond/ir/ui/menu.xml +++ b/trytond/ir/ui/menu.xml @@ -28,7 +28,7 @@ this repository contains the full copyright notices and license terms. --> Menu ir.ui.menu menu - + [('parent', '=', None)] diff --git a/trytond/ir/ui/view.py b/trytond/ir/ui/view.py index 86deccbd9b..99afc07fb6 100644 --- a/trytond/ir/ui/view.py +++ b/trytond/ir/ui/view.py @@ -5,7 +5,7 @@ import sys import logging from lxml import etree from trytond.model import ModelView, ModelSQL, fields -from trytond.backend import TableHandler +from trytond import backend from trytond.pyson import CONTEXT, Eval, Bool, PYSONDecoder from trytond.tools import safe_eval, file_open from trytond.transaction import Transaction @@ -69,6 +69,7 @@ class View(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor table = TableHandler(cursor, cls, module_name) @@ -312,6 +313,7 @@ class ViewTreeState(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor table = TableHandler(cursor, cls, module_name) diff --git a/trytond/model/fields/binary.py b/trytond/model/fields/binary.py index cabac594d1..94b278e2b0 100644 --- a/trytond/model/fields/binary.py +++ b/trytond/model/fields/binary.py @@ -1,8 +1,10 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from sql import Query, Expression -from trytond.model.fields.field import Field -from trytond.transaction import Transaction +from .field import Field, SQLType +from ...transaction import Transaction +from ...config import CONFIG class Binary(Field): @@ -13,8 +15,8 @@ class Binary(Field): def __init__(self, string='', help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, - on_change_with=None, depends=None, filename=None, order_field=None, - context=None, loading='lazy'): + on_change_with=None, depends=None, filename=None, context=None, + loading='lazy'): if filename is not None: self.filename = filename if depends is None: @@ -24,8 +26,7 @@ class Binary(Field): super(Binary, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) @staticmethod def get(ids, model, name, values=None): @@ -53,3 +54,21 @@ class Binary(Field): for i in ids: res.setdefault(i, default) return res + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + db_type = CONFIG['db_type'] + if db_type == 'postgresql' and value is not None: + import psycopg2 + return psycopg2.Binary(value) + return value + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'postgresql': + return SQLType('BYTEA', 'BYTEA') + elif db_type == 'mysql': + return SQLType('LONGBLOB', 'LONGBLOB') + return SQLType('BLOB', 'BLOB') diff --git a/trytond/model/fields/boolean.py b/trytond/model/fields/boolean.py index 7e785f99c2..0dc00f44ab 100644 --- a/trytond/model/fields/boolean.py +++ b/trytond/model/fields/boolean.py @@ -1,7 +1,7 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. -from trytond.model.fields.field import Field +from .field import Field, SQLType class Boolean(Field): @@ -12,11 +12,28 @@ class Boolean(Field): def __init__(self, string='', help='', readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, - depends=None, order_field=None, context=None, loading='eager'): + depends=None, context=None, loading='eager'): super(Boolean, self).__init__(string=string, help=help, required=False, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) __init__.__doc__ = Field.__init__.__doc__ + + def sql_type(self): + return SQLType('BOOL', 'BOOL') + + def _domain_add_null(self, column, operator, value, expression): + expression = super(Boolean, self)._domain_add_null( + column, operator, value, expression) + if operator in ('=', '!='): + conv = { + False: None, + None: False, + } + if value is False or value is None: + if operator == '=': + expression |= (column == conv[value]) + else: + expression &= (column != conv[value]) + return expression diff --git a/trytond/model/fields/char.py b/trytond/model/fields/char.py index 5f9f0bdb8a..64f1c1f478 100644 --- a/trytond/model/fields/char.py +++ b/trytond/model/fields/char.py @@ -1,7 +1,9 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from sql import Query, Expression -from trytond.model.fields.field import Field, size_validate +from ...config import CONFIG +from .field import Field, FieldTranslate, size_validate, SQLType def autocomplete_validate(value): @@ -9,7 +11,7 @@ def autocomplete_validate(value): assert isinstance(value, list), 'autocomplete must be a list' -class Char(Field): +class Char(FieldTranslate): ''' Define a char field (``unicode``). ''' @@ -18,8 +20,7 @@ class Char(Field): def __init__(self, string='', size=None, help='', required=False, readonly=False, domain=None, states=None, translate=False, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading=None, - autocomplete=None): + context=None, loading=None, autocomplete=None): ''' :param translate: A boolean. If ``True`` the field is translatable. :param size: A integer. If set defines the maximum size of the values. @@ -29,8 +30,7 @@ class Char(Field): super(Char, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.__autocomplete = None self.autocomplete = autocomplete if autocomplete else None self.translate = translate @@ -55,3 +55,22 @@ class Char(Field): self.__size = value size = property(_get_size, _set_size) + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + elif isinstance(value, str): + return unicode(value, 'utf-8') + assert isinstance(value, unicode) + return value + + def sql_type(self): + db_type = CONFIG['db_type'] + if self.size and db_type != 'sqlite': + return SQLType('VARCHAR', 'VARCHAR(%s)' % self.size) + elif db_type == 'mysql': + return SQLType('CHAR', 'VARCHAR(255)') + return SQLType('VARCHAR', 'VARCHAR') diff --git a/trytond/model/fields/date.py b/trytond/model/fields/date.py index 7102d38f0f..f769e9dd0c 100644 --- a/trytond/model/fields/date.py +++ b/trytond/model/fields/date.py @@ -1,7 +1,10 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +import datetime +from sql import Query, Expression -from trytond.model.fields.field import Field +from ...config import CONFIG +from .field import Field, SQLType class Date(Field): @@ -10,6 +13,27 @@ class Date(Field): ''' _type = 'date' + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + if isinstance(value, basestring): + year, month, day = map(int, value.split("-", 2)) + return datetime.date(year, month, day) + + assert(isinstance(value, datetime.date)) + # Allow datetime with min time for XML-RPC + # datetime must be tested separately because datetime is a + # subclass of date + assert(not isinstance(value, datetime.datetime) + or value.time() == datetime.time()) + return value + + def sql_type(self): + return SQLType('DATE', 'DATE') + class DateTime(Field): ''' @@ -20,22 +44,93 @@ class DateTime(Field): def __init__(self, string='', format='%H:%M:%S', help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='eager'): + context=None, loading='eager'): ''' :param format: The validation format as used by strftime. ''' super(DateTime, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.format = format __init__.__doc__ += Field.__init__.__doc__ + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if not value: + return None + if isinstance(value, basestring): + datepart, timepart = value.split(" ") + year, month, day = map(int, datepart.split("-", 2)) + hours, minutes, seconds = map(int, timepart.split(":")) + return datetime.datetime(year, month, day, hours, minutes, seconds) + assert(isinstance(value, datetime.datetime)) + return value.replace(microsecond=0) + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'sqlite': + return SQLType('TIMESTAMP', 'TIMESTAMP') + elif db_type == 'mysql': + return SQLType('TIMESTAMP', 'TIMESTAMP NULL') + return SQLType('TIMESTAMP', 'TIMESTAMP(0)') + + +class Timestamp(Field): + ''' + Define a timestamp field (``datetime``). + ''' + _type = 'timestamp' + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + if isinstance(value, basestring): + datepart, timepart = value.split(" ") + year, month, day = map(int, datepart.split("-", 2)) + timepart_full = timepart.split(".", 1) + hours, minutes, seconds = map(int, timepart_full[0].split(":")) + if len(timepart_full) == 2: + microseconds = int(timepart_full[1]) + else: + microseconds = 0 + return datetime.datetime(year, month, day, hours, minutes, seconds, + microseconds) + assert(isinstance(value, datetime.datetime)) + return value + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'sqlite': + return SQLType('TIMESTAMP', 'TIMESTAMP') + elif db_type == 'mysql': + return SQLType('TIMESTAMP', 'TIMESTAMP NULL') + return SQLType('TIMESTAMP', 'TIMESTAMP(6)') + class Time(DateTime): ''' Define a time field (``time``). ''' _type = 'time' + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + if isinstance(value, basestring): + hours, minutes, seconds = map(int, value.split(":")) + return datetime.time(hours, minutes, seconds) + assert(isinstance(value, datetime.time)) + return value.replace(microsecond=0) + + def sql_type(self): + return SQLType('TIME', 'TIME') diff --git a/trytond/model/fields/dict.py b/trytond/model/fields/dict.py index 42076d769a..a775916279 100644 --- a/trytond/model/fields/dict.py +++ b/trytond/model/fields/dict.py @@ -4,9 +4,10 @@ try: import simplejson as json except ImportError: import json +from sql import Query, Expression -from .field import Field -from trytond.protocols.jsonrpc import object_hook +from .field import Field, SQLType +from ...protocols.jsonrpc import object_hook, JSONEncoder class Dict(Field): @@ -16,10 +17,10 @@ class Dict(Field): def __init__(self, schema_model, string='', help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='lazy'): + context=None, loading='lazy'): super(Dict, self).__init__(string, help, required, readonly, domain, - states, select, on_change, on_change_with, depends, order_field, - context, loading) + states, select, on_change, on_change_with, depends, context, + loading) self.schema_model = schema_model def get(self, ids, model, name, values=None): @@ -29,3 +30,15 @@ class Dict(Field): dicts[value['id']] = json.loads(value[name], object_hook=object_hook) return dicts + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + assert isinstance(value, dict) + return json.dumps(value, cls=JSONEncoder) + + def sql_type(self): + return SQLType('TEXT', 'TEXT') diff --git a/trytond/model/fields/field.py b/trytond/model/fields/field.py index 3c6bd34a41..83f4303214 100644 --- a/trytond/model/fields/field.py +++ b/trytond/model/fields/field.py @@ -1,8 +1,14 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from collections import namedtuple +from sql import operators, Column, Literal, Select, CombiningQuery +from sql.conditionals import Coalesce, NullIf +from sql.operators import Concat from trytond.pyson import PYSON from trytond.const import OPERATORS +from trytond.transaction import Transaction +from trytond.pool import Pool def domain_validate(value): @@ -64,13 +70,28 @@ def size_validate(value): assert value.types() == set([int]), \ 'size must return integer' +SQL_OPERATORS = { + '=': operators.Equal, + '!=': operators.NotEqual, + 'like': operators.Like, + 'not like': operators.NotLike, + 'ilike': operators.ILike, + 'not ilike': operators.NotILike, + 'in': operators.In, + 'not in': operators.NotIn, + '<=': operators.LessEqual, + '>=': operators.GreaterEqual, + '<': operators.Less, + '>': operators.Greater, + } + class Field(object): _type = None def __init__(self, string='', help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, - on_change_with=None, depends=None, order_field=None, context=None, + on_change_with=None, depends=None, context=None, loading='eager'): ''' :param string: A string for label of the field. @@ -92,8 +113,6 @@ class Field(object): defined the other way around. The list contains all the fields that must update the current field. :param depends: A list of field name on which this one depends. - :param order_field: A string. If set it will use the string when - ordering records on the field. :param context: A dictionary which will be given to open the relation fields. :param loading: Define how the field must be loaded: @@ -115,7 +134,6 @@ class Field(object): self.on_change_with = on_change_with self.__depends = None self.depends = depends or [] - self.order_field = order_field self.__context = None self.context = context or {} assert loading in ('lazy', 'eager'), \ @@ -187,3 +205,136 @@ class Field(object): if inst._values is None: inst._values = {} inst._values[self.name] = value + + @staticmethod + def sql_format(value): + return value + + def sql_type(self): + raise NotImplementedError + + def _domain_value(self, operator, value): + if isinstance(value, (Select, CombiningQuery)): + return value + if operator in ('in', 'not in'): + return [self.sql_format(v) for v in value if v is not None] + else: + return self.sql_format(value) + + def _domain_add_null(self, column, operator, value, expression): + if operator in ('in', 'not in'): + if (not isinstance(value, (Select, CombiningQuery)) + and any(v is None for v in value)): + if operator == 'in': + expression |= (column == None) + else: + expression &= (column != None) + return expression + + def convert_domain(self, domain, tables, Model): + "Return a SQL expression for the domain using tables" + table, _ = tables[None] + name, operator, value = domain + Operator = SQL_OPERATORS[operator] + column = Column(table, name) + expression = Operator(column, self._domain_value(operator, value)) + if isinstance(expression, operators.In) and not expression.right: + expression = Literal(False) + elif isinstance(expression, operators.NotIn) and not expression.right: + expression = Literal(True) + expression = self._domain_add_null(column, operator, value, expression) + return expression + + def convert_order(self, name, tables, Model): + "Return a SQL expression to order" + table, _ = tables[None] + method = getattr(Model, 'order_%s' % name, None) + if method: + return method(tables) + else: + return [Column(table, name)] + + +class FieldTranslate(Field): + + def __get_translation_join(self, Model, name, + translation, model, table): + language = Transaction().language + if Model.__name__ == 'ir.model': + return table.join(translation, 'LEFT', + condition=(translation.name == Concat(table.model, name)) + & (translation.res_id == None) + & (translation.lang == language) + & (translation.type == 'model') + & (translation.fuzzy == False)) + elif Model.__name__ == 'ir.model.field': + if name == 'field_description': + type_ = 'field' + else: + type_ = 'help' + return table.join(model, 'LEFT', + condition=model.id == table.model).join( + translation, 'LEFT', + condition=(translation.name == Concat(Concat( + model.model, ','), table.name)) + & (translation.res_id == None) + & (translation.lang == language) + & (translation.type == type_) + & (translation.fuzzy == False)) + else: + return table.join(translation, 'LEFT', + condition=(translation.res_id == table.id) + & (translation.name == '%s,%s' % (Model.__name__, name)) + & (translation.lang == language) + & (translation.type == 'model') + & (translation.fuzzy == False)) + + def convert_domain(self, domain, tables, Model): + pool = Pool() + Translation = pool.get('ir.translation') + IrModel = pool.get('ir.model') + if not self.translate: + return super(FieldTranslate, self).convert_domain( + domain, tables, Model) + + table = Model.__table__() + translation = Translation.__table__() + model = IrModel.__table__() + name, operator, value = domain + join = self.__get_translation_join(Model, name, + translation, model, table) + Operator = SQL_OPERATORS[operator] + column = Coalesce(NullIf(translation.value, ''), + Column(table, name)) + where = Operator(column, self._domain_value(operator, value)) + if isinstance(where, operators.In) and not where.right: + where = Literal(False) + elif isinstance(where, operators.NotIn) and not where.right: + where = Literal(True) + where = self._domain_add_null(column, operator, value, where) + return tables[None][0].id.in_(join.select(table.id, where=where)) + + def convert_order(self, name, tables, Model): + pool = Pool() + Translation = pool.get('ir.translation') + IrModel = pool.get('ir.model') + if not self.translate: + return super(FieldTranslate, self).convert_order(name, tables, + Model) + + table, _ = tables[None] + key = name + '.translation' + if key not in tables: + translation = Translation.__table__() + model = IrModel.__table__() + join = self.__get_translation_join(Model, name, + translation, model, table) + tables[key] = { + None: (join.right, join.condition), + } + else: + translation, _ = tables[key][None] + + return [Coalesce(NullIf(translation.value, ''), Column(table, name))] + +SQLType = namedtuple('SQLType', 'base type') diff --git a/trytond/model/fields/float.py b/trytond/model/fields/float.py index a5d5c863ad..1041f0a50c 100644 --- a/trytond/model/fields/float.py +++ b/trytond/model/fields/float.py @@ -1,8 +1,10 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from sql import Query, Expression -from trytond.model.fields.field import Field -from trytond.pyson import PYSON +from ...config import CONFIG +from .field import Field, SQLType +from ...pyson import PYSON def digits_validate(value): @@ -25,7 +27,7 @@ class Float(Field): def __init__(self, string='', digits=None, help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='eager'): + context=None, loading='eager'): ''' :param digits: a list of two integers defining the total of digits and the number of decimals of the float. @@ -33,8 +35,7 @@ class Float(Field): super(Float, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.__digits = None self.digits = digits @@ -48,3 +49,20 @@ class Float(Field): self.__digits = value digits = property(_get_digits, _set_digits) + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + return float(value) + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'postgresql': + return SQLType('FLOAT8', 'FLOAT8') + elif db_type == 'mysql': + return SQLType('DOUBLE', 'DOUBLE(255, 15)') + else: + return SQLType('FLOAT', 'FLOAT') diff --git a/trytond/model/fields/function.py b/trytond/model/fields/function.py index 1971cad0bb..2ebc511963 100644 --- a/trytond/model/fields/function.py +++ b/trytond/model/fields/function.py @@ -59,14 +59,11 @@ class Function(Field): return setattr(self._field, name, value) - def search(self, model, name, clause): - ''' - Call the searcher. - Return a list of clauses. - ''' + def convert_domain(self, domain, tables, Model): + name, operator, value = domain[:3] if not self.searcher: - model.raise_user_error('search_function_missing', name) - return getattr(model, self.searcher)(name, tuple(clause)) + Model.raise_user_error('search_function_missing', name) + return getattr(Model, self.searcher)(name, domain) def get(self, ids, Model, name, values=None): ''' diff --git a/trytond/model/fields/integer.py b/trytond/model/fields/integer.py index 72f1d08cc1..2ba0694b23 100644 --- a/trytond/model/fields/integer.py +++ b/trytond/model/fields/integer.py @@ -1,7 +1,9 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from sql import Query, Expression -from trytond.model.fields.field import Field +from ...config import CONFIG +from .field import Field, SQLType class Integer(Field): @@ -10,9 +12,32 @@ class Integer(Field): ''' _type = 'integer' + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'postgresql': + return SQLType('INT4', 'INT4') + elif db_type == 'mysql': + return SQLType('SIGNED INTEGER', 'BIGINT') + else: + return SQLType('INTEGER', 'INTEGER') + + def sql_format(self, value): + db_type = CONFIG['db_type'] + if (db_type == 'sqlite' + and value is not None + and not isinstance(value, (Query, Expression))): + value = int(value) + return super(Integer, self).sql_format(value) + class BigInteger(Integer): ''' Define an integer field (``long``). ''' _type = 'biginteger' + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'postgresql': + return SQLType('INT8', 'INT8') + return super(BigInteger, self).sql_type() diff --git a/trytond/model/fields/many2many.py b/trytond/model/fields/many2many.py index 954af80f22..8a94044714 100644 --- a/trytond/model/fields/many2many.py +++ b/trytond/model/fields/many2many.py @@ -1,9 +1,12 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from itertools import chain -from trytond.model.fields.field import Field, size_validate -from trytond.transaction import Transaction -from trytond.pool import Pool +from sql import Cast, Literal, Column +from sql.functions import Substring, Position + +from .field import Field, size_validate +from ...transaction import Transaction +from ...pool import Pool class Many2Many(Field): @@ -15,8 +18,7 @@ class Many2Many(Field): def __init__(self, relation_name, origin, target, string='', order=None, datetime_field=None, size=None, help='', required=False, readonly=False, domain=None, states=None, on_change=None, - on_change_with=None, depends=None, order_field=None, context=None, - loading='lazy'): + on_change_with=None, depends=None, context=None, loading='lazy'): ''' :param relation_name: The name of the relation model or the name of the target model for ModelView only. @@ -36,8 +38,7 @@ class Many2Many(Field): super(Many2Many, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.relation_name = relation_name self.origin = origin self.target = target @@ -236,3 +237,94 @@ class Many2Many(Field): return Target(data) value = [instance(x) for x in (value or [])] super(Many2Many, self).__set__(inst, value) + + def convert_domain_child(self, domain, tables): + Target = self.get_target() + table, _ = tables[None] + name, operator, ids = domain + + def get_child(ids): + if not ids: + return [] + children = Target.search([ + (name, 'in', ids), + (name, '!=', None), + ], order=[]) + child_ids = get_child([c.id for c in children]) + return ids + child_ids + expression = table.id.in_(ids + get_child(ids)) + if operator == 'not child_of': + return ~expression + return expression + + def convert_domain(self, domain, tables, Model): + pool = Pool() + Target = self.get_target() + Relation = pool.get(self.relation_name) + relation = Relation.__table__() + table, _ = tables[None] + name, operator, value = domain[:3] + + origin_field = Relation._fields[self.origin] + origin = Column(relation, self.origin) + origin_where = None + if origin_field._type == 'reference': + origin_where = origin.like(Model.__name__ + ',%') + origin = Cast(Substring(origin, + Position(',', origin) + Literal(1)), + Relation.id.sql_type().base) + + if '.' not in name: + if operator in ('child_of', 'not child_of'): + if Target != Model: + query = Target.search([(domain[3], 'child_of', value)], + order=[], query=True) + where = (Column(relation, self.target).in_(query) + & (Column(relation, self.origin) != None)) + if origin_where: + where &= origin_where + query = relation.select(origin, where=where) + expression = table.id.in_(query) + if operator == 'not child_of': + return ~expression + return expression + if isinstance(value, basestring): + targets = Target.search([('rec_name', 'ilike', value)], + order=[]) + ids = [t.id for t in targets] + elif not isinstance(value, (list, tuple)): + ids = [value] + else: + ids = value + if not ids: + expression = table.id.in_([None]) + if operator == 'not child_of': + return ~expression + return expression + else: + return self.convert_domain_child( + (name, operator, ids), tables) + + if value is None: + where = origin != value + if origin_where: + where &= origin_where + query = relation.select(origin, where=where) + expression = ~table.id.in_(query) + if operator == '!=': + return ~expression + return expression + else: + if isinstance(value, basestring): + target_name = 'rec_name' + else: + target_name = 'id' + else: + _, target_name = name.split('.', 1) + target_domain = [(target_name,) + tuple(domain[1:])] + query = Target.search(target_domain, order=[], query=True) + where = Column(relation, self.target).in_(query) + if origin_where: + where &= origin_where + query = relation.select(origin, where=where) + return table.id.in_(query) diff --git a/trytond/model/fields/many2one.py b/trytond/model/fields/many2one.py index 5616c44f4e..8506f4b1c0 100644 --- a/trytond/model/fields/many2one.py +++ b/trytond/model/fields/many2one.py @@ -1,9 +1,14 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from types import NoneType +from sql import Column, Query, Expression +from sql.operators import Or -from trytond.model.fields.field import Field -from trytond.pool import Pool +from .field import Field, SQLType +from ...pool import Pool +from ...config import CONFIG +from ...tools import reduce_ids +from ...transaction import Transaction class Many2One(Field): @@ -16,7 +21,7 @@ class Many2One(Field): ondelete='SET NULL', datetime_field=None, help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='eager'): + context=None, loading='eager'): ''' :param model_name: The name of the target model. :param left: The name of the field to store the left value for @@ -41,8 +46,7 @@ class Many2One(Field): super(Many2One, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.model_name = model_name self.left = left self.right = right @@ -71,3 +75,124 @@ class Many2One(Field): value = Target(value) assert isinstance(value, (Target, NoneType)) super(Many2One, self).__set__(inst, value) + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + assert value is not False + return int(value) + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'postgresql': + return SQLType('INT4', 'INT4') + elif db_type == 'mysql': + return SQLType('SIGNED INTEGER', 'BIGINT') + else: + return SQLType('INTEGER', 'INTEGER') + + def convert_domain_child_mptt(self, domain, tables): + cursor = Transaction().cursor + table, _ = tables[None] + name, operator, ids = domain + red_sql = reduce_ids(table.id, ids) + left = Column(table, self.left) + right = Column(table, self.right) + cursor.execute(*table.select(left, right, where=red_sql)) + where = Or() + for l, r in cursor.fetchall(): + where.append((left >= l) & (right <= r)) + expression = table.id.in_(table.select(table.id, where=where)) + if operator == 'not child_of': + return ~expression + return expression + + def convert_domain_child(self, domain, tables): + Target = self.get_target() + table, _ = tables[None] + name, operator, ids = domain + + def get_child(ids): + if not ids: + return [] + children = Target.search([ + (name, 'in', ids), + (name, '!=', None), + ], order=[]) + child_ids = get_child([c.id for c in children]) + return ids + child_ids + expression = table.id.in_(ids + get_child(ids)) + if operator == 'not child_of': + return ~expression + return expression + + def convert_domain(self, domain, tables, Model): + Target = self.get_target() + table, _ = tables[None] + name, operator, value = domain[:3] + column = Column(table, name.split('.', 1)[0]) + if '.' not in name: + if operator in ('child_of', 'not child_of'): + if Target != Model: + query = Target.search([(domain[3], 'child_of', value)], + order=[], query=True) + expression = column.in_(query) + if operator == 'not child_of': + return ~expression + return expression + + if isinstance(value, basestring): + targets = Target.search([('rec_name', 'ilike', value)], + order=[]) + ids = [t.id for t in targets] + elif not isinstance(value, (list, tuple)): + ids = [value] + else: + ids = value + if not ids: + expression = column.in_([None]) + if operator == 'not child_of': + return ~expression + return expression + elif self.left and self.right: + return self.convert_domain_child_mptt( + (name, operator, ids), tables) + else: + return self.convert_domain_child( + (name, operator, ids), tables) + + if not isinstance(value, basestring): + return super(Many2One, self).convert_domain(domain, tables, + Model) + else: + target_name = 'rec_name' + else: + _, target_name = name.split('.', 1) + target_domain = [(target_name,) + tuple(domain[1:])] + if 'active' in Target._fields: + target_domain.append(('active', 'in', [True, False])) + query = Target.search(target_domain, order=[], query=True) + return column.in_(query) + + def convert_order(self, name, tables, Model): + Target = self.get_target() + + oname = 'id' + if Target._rec_name in Target._fields: + oname = Target._rec_name + if Target._order_name in Target._fields: + oname = Target._order_name + + ofield = Target._fields[oname] + table, _ = tables[None] + target_tables = tables.get(name) + if target_tables is None: + target = Target.__table__() + target_tables = { + None: (target, target.id == Column(table, name)), + } + tables[name] = target_tables + return ofield.convert_order(oname, target_tables, Target) diff --git a/trytond/model/fields/numeric.py b/trytond/model/fields/numeric.py index f7171aacc8..38b87d7843 100644 --- a/trytond/model/fields/numeric.py +++ b/trytond/model/fields/numeric.py @@ -1,7 +1,11 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from decimal import Decimal +from sql import Query, Expression -from trytond.model.fields.float import Float +from ...config import CONFIG +from .field import SQLType +from .float import Float class Numeric(Float): @@ -9,3 +13,20 @@ class Numeric(Float): Define a numeric field (``decimal``). ''' _type = 'numeric' + + @staticmethod + def sql_format(value): + if isinstance(value, (Query, Expression)): + return value + if value is None: + return None + if isinstance(value, (int, long)): + value = Decimal(str(value)) + assert isinstance(value, Decimal) + return value + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'mysql': + return SQLType('DECIMAL', 'DECIMAL(65, 30)') + return SQLType('NUMERIC', 'NUMERIC') diff --git a/trytond/model/fields/one2many.py b/trytond/model/fields/one2many.py index 177001b786..bb9f451eac 100644 --- a/trytond/model/fields/one2many.py +++ b/trytond/model/fields/one2many.py @@ -1,9 +1,12 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from itertools import chain -from trytond.model.fields.field import Field, size_validate -from trytond.transaction import Transaction -from trytond.pool import Pool +from sql import Cast, Literal, Column +from sql.functions import Substring, Position + +from .field import Field, size_validate +from ...transaction import Transaction +from ...pool import Pool def add_remove_validate(value): @@ -21,7 +24,7 @@ class One2Many(Field): order=None, datetime_field=None, size=None, help='', required=False, readonly=False, domain=None, states=None, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='lazy'): + context=None, loading='lazy'): ''' :param model_name: The name of the target model. :param field: The name of the field that handle the reverse many2one or @@ -42,8 +45,7 @@ class One2Many(Field): super(One2Many, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.model_name = model_name self.field = field self.__add_remove = None @@ -213,3 +215,43 @@ class One2Many(Field): return Target(data) value = [instance(x) for x in (value or [])] super(One2Many, self).__set__(inst, value) + + def convert_domain(self, domain, tables, Model): + Target = self.get_target() + target = Target.__table__() + table, _ = tables[None] + name, operator, value = domain[:3] + + origin_field = Target._fields[self.field] + origin = Column(target, self.field) + origin_where = None + if origin_field._type == 'reference': + origin_where = origin.like(Model.__name__ + ',%') + origin = Cast(Substring(origin, + Position(',', origin) + Literal(1)), + Target.id.sql_type().base) + + if '.' not in name: + if value is None: + where = origin != value + if origin_where: + where &= origin_where + query = target.select(origin, where=where) + expression = ~table.id.in_(query) + if operator == '!=': + return ~expression + return expression + else: + if isinstance(value, basestring): + target_name = 'rec_name' + else: + target_name = 'id' + else: + _, target_name = name.split('.', 1) + target_domain = [(target_name,) + tuple(domain[1:])] + query = Target.search(target_domain, order=[], query=True) + where = target.id.in_(query) + if origin_where: + where &= origin_where + query = target.select(origin, where=where) + return table.id.in_(query) diff --git a/trytond/model/fields/property.py b/trytond/model/fields/property.py index 182c3c6efc..568b845b14 100644 --- a/trytond/model/fields/property.py +++ b/trytond/model/fields/property.py @@ -2,11 +2,15 @@ #this repository contains the full copyright notices and license terms. import copy -from trytond.model.fields.function import Function -from trytond.model.fields.field import Field -from trytond import backend -from trytond.transaction import Transaction -from trytond.pool import Pool +from sql import Cast, Literal +from sql.functions import Substring, Position +from sql.conditionals import Case + +from .function import Function +from .field import Field, SQL_OPERATORS +from .numeric import Numeric +from ...transaction import Transaction +from ...pool import Pool class Property(Function): @@ -60,59 +64,41 @@ class Property(Function): prop_value = None Property.set(name, model.__name__, ids, prop_value) - def search(self, model, name, clause): - ''' - :param model: The model. - :param clause: The search domain clause. See ModelStorage.search - :return: New list of domain. - ''' + def convert_domain(self, domain, tables, Model): pool = Pool() Rule = pool.get('ir.rule') Property = pool.get('ir.property') - Model = pool.get('ir.model') + IrModel = pool.get('ir.model') Field = pool.get('ir.model.field') cursor = Transaction().cursor - field_class = backend.FIELDS[self._type] - if not field_class: - return [] - sql_type = field_class.sql_type(model._fields[name]) - if not sql_type: - return [] - sql_type = sql_type[0] + name, operator, value = domain - property_query, property_val = Rule.domain_get('ir.property') + sql_type = self._field.sql_type().base - property_clause = '' - if property_query: - property_clause = 'AND ' + property_query + property_cond = Rule.domain_get('ir.property') + + property_ = Property.__table__() + model_field = Field.__table__() + model = IrModel.__table__() #Fetch res ids that comply with the domain - cursor.execute( - 'SELECT CAST(' - 'SUBSTR("' + Property._table + '".res, ' - 'POSITION(\',\' IN "' + Property._table + '".res) + 1) ' - 'AS INTEGER), ' - '"' + Property._table + '".id ' - 'FROM "' + Property._table + '" ' - 'JOIN "' + Field._table + '" ON ' - '("' + Field._table + '"' + - '.id = "' + Property._table + '".field) ' - 'JOIN "' + Model._table + '" ON ' - '("' + Model._table + - '".id = "' + Field._table + '".model) ' - 'WHERE ' - 'CASE WHEN "' + - Model._table + '".model = %s ' - 'AND "' + Field._table + '".name = %s ' - + property_clause + - ' THEN ' + - self.get_condition(sql_type, clause) + - ' ELSE ' - '%s ' - 'END', - [model.__name__, name] + property_val + - self.get_condition_args(clause) + [False]) + join = property_.join(model_field, + condition=model_field.id == property_.field) + join = join.join(model, + condition=model.id == model_field.model) + cond = ((model.model == Model.__name__) + & (model_field.name == name)) + if property_cond: + cond &= property_.id.in_(property_cond) + cursor.execute(*join.select( + Cast(Substring(property_.res, + Position(',', property_.res) + Literal(1)), + Model.id.sql_type().base), + property_.id, + where=Case((cond, self.get_condition(sql_type, domain, + property_)), + else_=False))) props = cursor.fetchall() default = None @@ -121,25 +107,24 @@ class Property(Function): default = prop[1] break - if not default \ - or ((clause[2] is False or clause[2] is None) - and clause[1] in ['=', '!=']) \ - or (clause[1] in ['not like', 'not ilike', 'not in', '!=']): - operator = 'in' # default operator - if (((clause[2] is False or clause[2] is None) - and clause[1] == '=') - or ((clause[2] is not False and clause[2] is not None) - and clause[1] in [ + if (not default + or ((value is False or value is None) + and operator in ['=', '!=']) + or (operator in ['not like', 'not ilike', 'not in', '!='])): + dom_operator = 'in' # default operator + if (((value is False or value is None) and operator == '=') + or ((value is not False and value is not None) + and operator in [ 'not like', 'not ilike', 'not in', '!='])): - operator = 'not in' - return [('id', operator, [x[0] for x in props])] + dom_operator = 'not in' + return [('id', dom_operator, [x[0] for x in props])] #Fetch the res ids that doesn't use the default value - cursor.execute( - "SELECT CAST(SUBSTR(res, POSITION(',' IN res) + 1) AS integer) " - 'FROM "' + Property._table + '"' - 'WHERE ' + property_query + ' AND res is not null', - property_val) + cursor.execute(*property_.select( + Cast(Substring(property_.res, + Position(',', property_.res) + Literal(1)), + Model.id.sql_type().base), + where=property_cond & (property_.res != None))) fetchall = cursor.fetchall() if not fetchall: @@ -148,7 +133,7 @@ class Property(Function): else: other_ids = [x[0] for x in cursor.fetchall()] - res_ids = model.search(['OR', + res_ids = Model.search(['OR', ('id', 'in', [x[0] for x in props]), ('id', 'not in', other_ids) ]) @@ -156,36 +141,35 @@ class Property(Function): return [('id', 'in', res_ids)] @staticmethod - def get_condition(sql_type, clause): - operator = '%s' - if sql_type == 'NUMERIC': - operator = 'CAST(%s AS NUMERIC)' + def get_condition(sql_type, clause, table): + operator = clause[1] + value = clause[2] + + numeric = Numeric('numeric') + if sql_type == numeric.sql_type().base and value: + if isinstance(value, (list, tuple)): + value = [Cast(v, sql_type) for v in value] + else: + value = Cast(value, sql_type) + + if value is None: + value = False + + column = Cast(Substring(table.value, + Position(',', table.value) + Literal(1)), + sql_type) + Operator = SQL_OPERATORS[operator] - value = "SUBSTR(value, POSITION(',' IN value) + 1)" # All negative clauses will be negated later - if clause[1] in ('in', 'not in'): - operator = operator % '%%s' - return ("(CAST(" + value + " AS %s) IN (" - + ",".join((operator,) * len(clause[2])) + ")) ") % sql_type - elif ((clause[2] is False or clause[2] is None) - and clause[1] in ['=', '!=']): - return "((CAST(" + value + " AS %s) IS NULL) = %%s) " % sql_type - elif clause[1] in ['not like', 'not ilike']: - return "(CAST(" + value + " AS %s) %s %s) " % \ - (sql_type, clause[1].split()[1], operator) - elif clause[1] == '!=': - return "(CAST(" + value + " AS %s) = %s) " % (sql_type, operator) - else: - return "(CAST(" + value + " AS %s) %s %s) " % \ - (sql_type, clause[1], operator) - - @staticmethod - def get_condition_args(clause): - res = [] - if clause[1] in ('in', 'not in'): - res.extend(clause[2]) - elif clause[2] is None: - res.append(False) - else: - res.append(clause[2]) - return res + if operator in ('in', 'not in'): + return column.in_(value) + elif ((value is False or value is None) + and operator in ('=', '!=')): + return (column == None) == value + elif operator == 'not like': + return column.like(value) + elif operator == 'not ilike': + return column.ilike(value) + elif operator == '!=': + return column == value + return Operator(column, value) diff --git a/trytond/model/fields/reference.py b/trytond/model/fields/reference.py index 3f07569b64..62c4badf32 100644 --- a/trytond/model/fields/reference.py +++ b/trytond/model/fields/reference.py @@ -2,10 +2,14 @@ #this repository contains the full copyright notices and license terms. import contextlib from types import NoneType +from sql import Cast, Literal, Column, Query, Expression +from sql.functions import Substring, Position -from trytond.model.fields.field import Field -from trytond.transaction import Transaction -from trytond.pool import Pool +from .field import Field, SQLType +from .char import Char +from ...transaction import Transaction +from ...pool import Pool +from ...config import CONFIG class Reference(Field): @@ -17,7 +21,7 @@ class Reference(Field): def __init__(self, string='', selection=None, selection_change_with=None, help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='lazy'): + context=None, loading='lazy'): ''' :param selection: A list or a function name that returns a list. The list must be a list of tuples. First member is an internal name @@ -26,8 +30,7 @@ class Reference(Field): super(Reference, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) self.selection = selection or None self.selection_change_with = selection_change_with @@ -93,3 +96,37 @@ class Reference(Field): Target = Pool().get(target) value = Target(id_) super(Reference, self).__set__(inst, value) + + @staticmethod + def sql_format(value): + if not isinstance(value, (basestring, Query, Expression)): + try: + value = '%s,%s' % tuple(value) + except TypeError: + pass + return Char.sql_format(value) + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'mysql': + return SQLType('CHAR', 'VARCHAR(255)') + return SQLType('VARCHAR', 'VARCHAR') + + def convert_domain(self, domain, tables, Model): + if '.' not in domain[0]: + return super(Reference, self).convert_domain(domain, tables, Model) + pool = Pool() + name, operator, value, target = domain[:4] + Target = pool.get(target) + table, _ = tables[None] + name, target_name = name.split('.', 1) + column = Column(table, name) + target_domain = [(target_name,) + tuple(domain[1:3]) + + tuple(domain[4:])] + if 'active' in Target._fields: + target_domain.append(('active', 'in', [True, False])) + query = Target.search(target_domain, order=[], query=True) + return (Cast(Substring(column, + Position(',', column) + Literal(1)), + Model.id.sql_type().base).in_(query) + & column.ilike(target + ',%')) diff --git a/trytond/model/fields/selection.py b/trytond/model/fields/selection.py index 03e2d1fbe5..ba44c00a6e 100644 --- a/trytond/model/fields/selection.py +++ b/trytond/model/fields/selection.py @@ -1,7 +1,10 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +from sql import Column +from sql.conditionals import Case -from trytond.model.fields.field import Field +from ...config import CONFIG +from .field import Field, SQLType class Selection(Field): @@ -14,7 +17,7 @@ class Selection(Field): selection_change_with=None, translate=True, help='', required=False, readonly=False, domain=None, states=None, select=False, on_change=None, on_change_with=None, depends=None, - order_field=None, context=None, loading='eager'): + context=None, loading='eager'): ''' :param selection: A list or a function name that returns a list. The list must be a list of tuples. First member is the value @@ -24,8 +27,7 @@ class Selection(Field): super(Selection, self).__init__(string=string, help=help, required=required, readonly=readonly, domain=domain, states=states, select=select, on_change=on_change, on_change_with=on_change_with, - depends=depends, order_field=order_field, context=context, - loading=loading) + depends=depends, context=context, loading=loading) if hasattr(selection, 'copy'): self.selection = selection.copy() else: @@ -34,3 +36,23 @@ class Selection(Field): self.sort = sort self.translate_selection = translate __init__.__doc__ += Field.__init__.__doc__ + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'mysql': + return SQLType('CHAR', 'VARCHAR(255)') + return SQLType('VARCHAR', 'VARCHAR') + + def convert_order(self, name, tables, Model): + if getattr(Model, 'order_%s' % name, None): + return super(Selection, self).convert_order(name, tables, Model) + + table, _ = tables[None] + selections = Model.fields_get([name])[name]['selection'] + if not isinstance(selections, (tuple, list)): + selections = getattr(Model, selections)() + column = Column(table, name) + whens = [] + for key, value in selections: + whens.append((column == key, value)) + return [Case(*whens, else_=column)] diff --git a/trytond/model/fields/sha.py b/trytond/model/fields/sha.py index ca05de4007..0209337866 100644 --- a/trytond/model/fields/sha.py +++ b/trytond/model/fields/sha.py @@ -1,11 +1,28 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. +import hashlib +from sql import Query, Expression -from trytond.model.fields.field import Field +from ...config import CONFIG +from .field import SQLType +from .char import Char -class Sha(Field): +class Sha(Char): ''' Define a sha field (``unicode``) of len 40. ''' _type = 'sha' + + def sql_format(self, value): + if value is not None and not isinstance(value, (Query, Expression)): + if isinstance(value, unicode): + value = value.encode('utf-8') + value = hashlib.sha1(value).hexdigest() + return super(Sha, self).sql_format(value) + + def sql_type(self): + db_type = CONFIG['db_type'] + if db_type == 'mysql': + return SQLType('CHAR', 'VARCHAR(40)') + return SQLType('VARCHAR', 'VARCHAR(40)') diff --git a/trytond/model/fields/text.py b/trytond/model/fields/text.py index db3f4e848d..6d2b37ece9 100644 --- a/trytond/model/fields/text.py +++ b/trytond/model/fields/text.py @@ -1,7 +1,8 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. -from trytond.model.fields.char import Char +from .field import SQLType +from .char import Char class Text(Char): @@ -9,3 +10,6 @@ class Text(Char): Define a text field (``unicode``). ''' _type = 'text' + + def sql_type(self): + return SQLType('TEXT', 'TEXT') diff --git a/trytond/model/model.py b/trytond/model/model.py index dfd80024e2..d105038510 100644 --- a/trytond/model/model.py +++ b/trytond/model/model.py @@ -109,200 +109,15 @@ class Model(WarningErrorMixin, URLMixin, PoolBase): super(Model, cls).__register__(module_name) pool = Pool() Translation = pool.get('ir.translation') - Property = pool.get('ir.property') + Model_ = pool.get('ir.model') + ModelField = pool.get('ir.model.field') - cursor = Transaction().cursor - # Add model in ir_model - cursor.execute("SELECT id FROM ir_model WHERE model = %s", - (cls.__name__,)) - model_id = None - if cursor.rowcount == -1 or cursor.rowcount is None: - data = cursor.fetchone() - if data: - model_id, = data - elif cursor.rowcount != 0: - model_id, = cursor.fetchone() - if not model_id: - cursor.execute("INSERT INTO ir_model " - "(model, name, info, module) VALUES (%s, %s, %s, %s)", - (cls.__name__, cls._get_name(), cls.__doc__, - module_name)) - Property._models_get_cache.clear() - cursor.execute("SELECT id FROM ir_model WHERE model = %s", - (cls.__name__,)) - (model_id,) = cursor.fetchone() - elif cls.__doc__: - cursor.execute('UPDATE ir_model ' - 'SET name = %s, ' - 'info = %s ' - 'WHERE id = %s', - (cls._get_name(), cls.__doc__, model_id)) + model_id = Model_.register(cls, module_name) + ModelField.register(cls, module_name, model_id) - # Update translation of model - if cls.__doc__: - name = cls.__name__ + ',name' - src = cls._get_name() - cursor.execute('SELECT id FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s ' - 'AND (res_id IS NULL OR res_id = %s)', - ('en_US', 'model', name, 0)) - trans_id = None - if cursor.rowcount == -1 or cursor.rowcount is None: - data = cursor.fetchone() - if data: - trans_id, = data - elif cursor.rowcount != 0: - trans_id, = cursor.fetchone() - src_md5 = Translation.get_src_md5(src) - if trans_id is None: - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (name, 'en_US', 'model', src, src_md5, '', module_name, - False)) - else: - cursor.execute('UPDATE ir_translation ' - 'SET src = %s, src_md5 = %s ' - 'WHERE id = %s', - (src, src_md5, trans_id)) - - # Add field in ir_model_field and update translation - cursor.execute('SELECT f.id AS id, f.name AS name, ' - 'f.field_description AS field_description, ' - 'f.ttype AS ttype, f.relation AS relation, ' - 'f.module as module, f.help AS help ' - 'FROM ir_model_field AS f, ir_model AS m ' - 'WHERE f.model = m.id ' - 'AND m.model = %s ', - (cls.__name__,)) - model_fields = {} - for field in cursor.dictfetchall(): - model_fields[field['name']] = field - - # Prefetch field translations - if cls._fields: - cursor.execute('SELECT id, name, src, type FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type IN (%s, %s, %s) ' - 'AND name IN ' - '(' + ','.join(('%s',) * len(cls._fields)) + ')', - ('en_US', 'field', 'help', 'selection') - + tuple([cls.__name__ + ',' + x for x in cls._fields])) - trans_fields = {} - trans_help = {} - trans_selection = {} - for trans in cursor.dictfetchall(): - if trans['type'] == 'field': - trans_fields[trans['name']] = trans - elif trans['type'] == 'help': - trans_help[trans['name']] = trans - elif trans['type'] == 'selection': - trans_selection.setdefault(trans['name'], {}) - trans_selection[trans['name']][trans['src']] = trans - - for field_name in cls._fields: - field = cls._fields[field_name] - relation = '' - if hasattr(field, 'model_name'): - relation = field.model_name - elif hasattr(field, 'relation_name'): - relation = field.relation_name - if field_name not in model_fields: - cursor.execute("INSERT INTO ir_model_field " - "(model, name, field_description, ttype, " - "relation, help, module) " - "VALUES (%s, %s, %s, %s, %s, %s, %s)", - (model_id, field_name, field.string, field._type, - relation, field.help, module_name)) - elif (model_fields[field_name]['field_description'] != field.string - or model_fields[field_name]['ttype'] != field._type - or model_fields[field_name]['relation'] != relation - or model_fields[field_name]['help'] != field.help): - cursor.execute('UPDATE ir_model_field ' - 'SET field_description = %s, ' - 'ttype = %s, ' - 'relation = %s, ' - 'help = %s ' - 'WHERE id = %s ', - (field.string, field._type, relation, - field.help, model_fields[field_name]['id'])) - trans_name = cls.__name__ + ',' + field_name - string_md5 = Translation.get_src_md5(field.string) - if trans_name not in trans_fields: - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (trans_name, 'en_US', 'field', field.string, - string_md5, '', module_name, False)) - elif trans_fields[trans_name]['src'] != field.string: - cursor.execute('UPDATE ir_translation ' - 'SET src = %s, src_md5 = %s ' - 'WHERE id = %s ', - (field.string, string_md5, trans_fields[trans_name]['id'])) - help_md5 = Translation.get_src_md5(field.help) - if trans_name not in trans_help: - if field.help: - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, ' - 'fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (trans_name, 'en_US', 'help', field.help, help_md5, '', - module_name, False)) - elif trans_help[trans_name]['src'] != field.help: - cursor.execute('UPDATE ir_translation ' - 'SET src = %s, src_md5 = %s ' - 'WHERE id = %s ', - (field.help, help_md5, trans_help[trans_name]['id'])) - if (hasattr(field, 'selection') - and isinstance(field.selection, (tuple, list)) - and ((hasattr(field, 'translate_selection') - and field.translate_selection) - or not hasattr(field, 'translate_selection'))): - for (_, val) in field.selection: - if (trans_name not in trans_selection - or val not in trans_selection[trans_name]): - val_md5 = Translation.get_src_md5(val) - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, ' - 'fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (trans_name, 'en_US', 'selection', val, val_md5, - '', module_name, False)) - # Clean ir_model_field from field that are no more existing. - for field_name in model_fields: - if model_fields[field_name]['module'] == module_name \ - and field_name not in cls._fields: - #XXX This delete field even when it is defined later - # in the module - cursor.execute('DELETE FROM ir_model_field ' - 'WHERE id = %s', - (model_fields[field_name]['id'],)) - - # Add error messages in ir_translation - cursor.execute('SELECT id, src FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s', - ('en_US', 'error', cls.__name__)) - trans_error = {} - for trans in cursor.dictfetchall(): - trans_error[trans['src']] = trans - - errors = cls._get_error_messages() - for error in set(errors): - if error not in trans_error: - error_md5 = Translation.get_src_md5(error) - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (cls.__name__, 'en_US', 'error', error, error_md5, '', - module_name, False)) - - @classmethod - def _get_error_messages(cls): - return cls._error_messages.values() + Translation.register_model(cls, module_name) + Translation.register_fields(cls, module_name) + Translation.register_error_messages(cls, module_name) @classmethod def default_get(cls, fields_names, with_rec_name=True): @@ -435,9 +250,9 @@ class Model(WarningErrorMixin, URLMixin, PoolBase): and getattr(cls._fields[field], arg): res[field][arg] = copy.copy(getattr(cls._fields[field], arg)) - if isinstance(cls._fields[field], - (fields.Function, fields.One2Many)) \ - and not cls._fields[field].order_field: + if (isinstance(cls._fields[field], + (fields.Function, fields.One2Many)) + and not getattr(cls, 'order_%s' % field, None)): res[field]['sortable'] = False if ((isinstance(cls._fields[field], fields.Function) and not cls._fields[field].searcher) diff --git a/trytond/model/modelsql.py b/trytond/model/modelsql.py index 1040916bcd..20e11ed54b 100644 --- a/trytond/model/modelsql.py +++ b/trytond/model/modelsql.py @@ -1,16 +1,19 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. import contextlib -import datetime import re from functools import reduce -from decimal import Decimal from itertools import islice, izip +from sql import Table, Column, Literal, Desc, Asc, Expression, Flavor +from sql.functions import Now, Extract +from sql.conditionals import Coalesce +from sql.operators import Or, And, Operator +from sql.aggregate import Count, Max + from trytond.model import ModelStorage from trytond.model import fields -from trytond.backend import FIELDS, TableHandler -from trytond.backend import DatabaseIntegrityError +from trytond import backend from trytond.tools import reduce_ids from trytond.const import OPERATORS, RECORD_CACHE_SIZE from trytond.transaction import Transaction @@ -43,8 +46,19 @@ class ModelSQL(ModelStorage): assert cls._table[-9:] != '__history', \ 'Model _table %s cannot end with "__history"' % cls._table + @classmethod + def __table__(cls): + return cls.table_query() or Table(cls._table) + + @classmethod + def __table_history__(cls): + if not cls._history: + raise ValueError('No history table') + return Table(cls._table + '__history') + @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(ModelSQL, cls).__register__(module_name) if cls.table_query(): @@ -57,116 +71,71 @@ class ModelSQL(ModelStorage): if cls._history: history_table = TableHandler(Transaction().cursor, cls, module_name, history=True) - timestamp_field = FIELDS['timestamp'] - integer_field = FIELDS['integer'] - logs = ( - ('create_date', timestamp_field.sql_type(None), - timestamp_field.sql_format, lambda *a: datetime.datetime.now(), - cls.create_date.string), - ('write_date', timestamp_field.sql_type(None), - timestamp_field.sql_format, None, cls.write_date.string), - ('create_uid', (integer_field.sql_type(None)[0], - 'INTEGER REFERENCES res_user ON DELETE SET NULL',), - integer_field.sql_format, lambda *a: 0, cls.create_uid.string), - ('write_uid', (integer_field.sql_type(None)[0], - 'INTEGER REFERENCES res_user ON DELETE SET NULL'), - integer_field.sql_format, None, cls.write_uid.string), - ) - for log in logs: - table.add_raw_column(log[0], log[1], log[2], - default_fun=log[3], migrate=False, string=log[4]) - if cls._history: - history_logs = ( - ('create_date', timestamp_field.sql_type(None), - timestamp_field.sql_format, cls.create_date.string), - ('write_date', timestamp_field.sql_type(None), - timestamp_field.sql_format, cls.write_date.string), - ('create_uid', (integer_field.sql_type(None)[0], - 'INTEGER REFERENCES res_user ON DELETE SET NULL',), - integer_field.sql_format, cls.create_uid.string), - ('write_uid', (integer_field.sql_type(None)[0], - 'INTEGER REFERENCES res_user ON DELETE SET NULL'), - integer_field.sql_format, cls.write_uid.string), - ) - for log in history_logs: - history_table.add_raw_column(log[0], log[1], log[2], - migrate=False, string=log[3]) history_table.index_action('id', action='add') for field_name, field in cls._fields.iteritems(): - default_fun = None - if field_name in ( - 'id', - 'write_uid', - 'write_date', - 'create_uid', - 'create_date', - 'rec_name', - ): + if field_name == 'id': continue + default_fun = None + try: + sql_type = field.sql_type() + except NotImplementedError: + continue + if field_name in cls._defaults: + default_fun = cls._defaults[field_name] - if not hasattr(field, 'set'): - if field_name in cls._defaults: - default_fun = cls._defaults[field_name] + def unpack_wrapper(fun): + def unpack_result(*a): + try: + # XXX ugly hack: some default fct try + # to access the non-existing table + result = fun(*a) + except Exception: + return None + clean_results = cls._clean_defaults( + {field_name: result}) + return clean_results[field_name] + return unpack_result + default_fun = unpack_wrapper(default_fun) - def unpack_wrapper(fun): - def unpack_result(*a): - try: - # XXX ugly hack: some default fct try - # to access the non-existing table - result = fun(*a) - except Exception: - return None - clean_results = cls._clean_defaults( - {field_name: result}) - return clean_results[field_name] - return unpack_result - default_fun = unpack_wrapper(default_fun) + if hasattr(field, 'size') and isinstance(field.size, int): + field_size = field.size + else: + field_size = None - if hasattr(field, 'size') and isinstance(field.size, int): - field_size = field.size + table.add_raw_column(field_name, sql_type, field.sql_format, + default_fun, field_size, string=field.string) + if cls._history: + history_table.add_raw_column(field_name, sql_type, None, + string=field.string) + + if isinstance(field, (fields.Integer, fields.Float)): + # migration from tryton 2.2 + table.db_default(field_name, None) + + if isinstance(field, (fields.Boolean)): + table.db_default(field_name, False) + + if isinstance(field, fields.Many2One): + if field.model_name in ('res.user', 'res.group'): + ref = field.model_name.replace('.', '_') else: - field_size = None - table.add_raw_column(field_name, - FIELDS[field._type].sql_type(field), - FIELDS[field._type].sql_format, default_fun, - field_size, string=field.string) - if cls._history: - history_table.add_raw_column(field_name, - FIELDS[field._type].sql_type(field), None, - string=field.string) + ref = pool.get(field.model_name)._table + table.add_fk(field_name, ref, field.ondelete) - if isinstance(field, (fields.Integer, fields.Float)): - # migration from tryton 2.2 - table.db_default(field_name, None) + table.index_action( + field_name, action=field.select and 'add' or 'remove') - if isinstance(field, (fields.Boolean)): - table.db_default(field_name, False) - - if isinstance(field, fields.Many2One): - if field.model_name in ('res.user', 'res.group'): - ref = field.model_name.replace('.', '_') - else: - ref = pool.get(field.model_name)._table - table.add_fk(field_name, ref, field.ondelete) - - table.index_action( - field_name, action=field.select and 'add' or 'remove') - - required = field.required - table.not_null_action( - field_name, action=required and 'add' or 'remove') - - elif not isinstance(field, (fields.One2Many, fields.Function, - fields.Many2Many)): - raise Exception('Unknow field type !') + required = field.required + table.not_null_action( + field_name, action=required and 'add' or 'remove') for field_name, field in cls._fields.iteritems(): if isinstance(field, fields.Many2One) \ and field.model_name == cls.__name__ \ and field.left and field.right: with Transaction().set_user(0): - cls._rebuild_tree(field_name, False, 0) + cls._rebuild_tree(field_name, None, 0) for ident, constraint, _ in cls._sql_constraints: table.add_constraint(ident, constraint) @@ -174,21 +143,24 @@ class ModelSQL(ModelStorage): if cls._history: cls._update_history_table() cursor = Transaction().cursor - cursor.execute('SELECT id FROM "' + cls._table + '"') + table = cls.__table__() + history_table = cls.__table_history__() + cursor.execute(*table.select(table.id)) if cursor.fetchone(): - cursor.execute('SELECT id FROM "' + cls._table + '__history"') + cursor.execute(*history_table.select(history_table.id)) if not cursor.fetchone(): - columns = ['"' + str(x) + '"' for x in cls._fields - if not hasattr(cls._fields[x], 'set')] - cursor.execute('INSERT INTO "' + cls._table + '__history" ' - '(' + ','.join(columns) + ') ' - 'SELECT ' + ','.join(columns) + - ' FROM "' + cls._table + '"') - cursor.execute('UPDATE "' + cls._table + '__history" ' - 'SET write_date = NULL') + columns = [n for n, f in cls._fields.iteritems() + if not hasattr(f, 'set')] + cursor.execute(*history_table.insert( + [Column(history_table, c) for c in columns], + table.select(*(Column(table, c) + for c in columns)))) + cursor.execute(*history_table.update( + [history_table.write_date], [None])) @classmethod def _update_history_table(cls): + TableHandler = backend.get('TableHandler') if cls._history: table = TableHandler(Transaction().cursor, cls) history_table = TableHandler(Transaction().cursor, cls, @@ -218,136 +190,190 @@ class ModelSQL(ModelStorage): ''' return None + @classmethod + def __raise_integrity_error(cls, exception, values, field_names=None): + pool = Pool() + if field_names is None: + field_names = cls._fields.keys() + for field_name in field_names: + if field_name not in cls._fields: + continue + field = cls._fields[field_name] + # Check required fields + if (field.required + and not hasattr(field, 'set') + and field_name not in ('create_uid', 'create_date')): + if values.get(field_name) is None: + cls.raise_user_error('required_field', + error_args=cls._get_error_args(field_name)) + if isinstance(field, fields.Many2One) and values.get(field_name): + Model = pool.get(field.model_name) + create_records = Transaction().create_records.get( + field.model_name, set()) + delete_records = Transaction().delete_records.get( + field.model_name, set()) + target_records = Model.search([ + ('id', '=', values[field_name]), + ], order=[]) + if not ((target_records + or (values[field_name] in create_records)) + and (values[field_name] not in delete_records)): + cls.raise_user_error('foreign_model_missing', + error_args=cls._get_error_args(field_name)) + for name, _, error in cls._sql_constraints: + if name in exception[0]: + cls.raise_user_error(error) + for name, error in cls._sql_error_messages.iteritems(): + if name in exception[0]: + cls.raise_user_error(error) + + @classmethod + def __insert_history(cls, ids, deleted=False): + transaction = Transaction() + cursor = transaction.cursor + in_max = cursor.IN_MAX + if not cls._history: + return + user = transaction.user + table = cls.__table__() + history = cls.__table_history__() + columns = [] + hcolumns = [] + if not deleted: + fields = cls._fields + else: + fields = { + 'id': cls.id, + 'write_uid': cls.write_uid, + 'write_date': cls.write_date, + } + for fname, field in sorted(fields.iteritems()): + if hasattr(field, 'set'): + continue + columns.append(Column(table, fname)) + hcolumns.append(Column(history, fname)) + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + if not deleted: + where = reduce_ids(table.id, sub_ids) + cursor.execute(*history.insert(hcolumns, + table.select(*columns, where=where))) + else: + cursor.execute(*history.insert(hcolumns, + [[id_, Now(), user] for id_ in sub_ids])) + + @classmethod + def __check_timestamp(cls, ids): + transaction = Transaction() + cursor = transaction.cursor + in_max = cursor.IN_MAX + table = cls.__table__() + if not transaction.timestamp: + return + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + where = Or() + for id_ in sub_ids: + try: + timestamp = transaction.timestamp.pop( + '%s,%s' % (cls.__name__, i)) + except KeyError: + continue + sql_type = fields.Numeric('timestamp').sql_type().base + where.append((table.id == id_) + & Extract('EPOCH', + Coalesce(table.write_date, table.create_date) + ).cast(sql_type) > timestamp) + if where: + cursor.execute(*table.select(table.id, where=where)) + if cursor.fetchone(): + raise ConcurrencyException( + 'Records were modified in the meanwhile') + @classmethod def create(cls, vlist): - super(ModelSQL, cls).create(vlist) - cursor = Transaction().cursor + DatabaseIntegrityError = backend.get('DatabaseIntegrityError') + transaction = Transaction() + cursor = transaction.cursor pool = Pool() Translation = pool.get('ir.translation') + in_max = cursor.IN_MAX + + super(ModelSQL, cls).create(vlist) if cls.table_query(): return False + table = cls.__table__() modified_fields = set() new_ids = [] vlist = [v.copy() for v in vlist] for values in vlist: # Clean values - for key in ('create_uid', 'create_date', 'write_uid', 'write_date', - 'id'): + for key in ('create_uid', 'create_date', + 'write_uid', 'write_date', 'id'): if key in values: del values[key] modified_fields |= set(values.keys()) # Get default values default = [] - for i in cls._fields.keys(): - if not i in values \ - and i not in ('create_uid', 'create_date', - 'write_uid', 'write_date'): - default.append(i) + for f in cls._fields.keys(): + if (f not in values + and f not in ('create_uid', 'create_date', + 'write_uid', 'write_date', 'id')): + default.append(f) - if len(default): + if default: defaults = cls.default_get(default, with_rec_name=False) - for field in defaults.keys(): - if '.' in field: - del defaults[field] - if field in ('create_uid', 'create_date', - 'write_uid', 'write_date'): - del defaults[field] - if field in values: - del defaults[field] values.update(cls._clean_defaults(defaults)) - (upd0, upd1, upd2) = ('', '', []) + insert_columns = [table.create_uid, table.create_date] + insert_values = [transaction.user, Now()] # Insert record for fname, value in values.iteritems(): field = cls._fields[fname] if not hasattr(field, 'set'): - upd0 = upd0 + ',"' + fname + '"' - upd1 = upd1 + ', %s' - upd2.append(FIELDS[field._type].sql_format(value)) - upd0 += ', create_uid, create_date' - upd1 += ', %s, %s' - upd2.append(Transaction().user) - upd2.append(datetime.datetime.now()) + insert_columns.append(Column(table, fname)) + insert_values.append(field.sql_format(value)) try: if cursor.has_returning(): - cursor.execute('INSERT INTO "' + cls._table + '" ' - '(' + upd0[1:] + ') ' - 'VALUES (' + upd1[1:] + ') RETURNING id', - tuple(upd2)) + cursor.execute(*table.insert(insert_columns, + [insert_values], [table.id])) id_new, = cursor.fetchone() else: id_new = cursor.nextid(cls._table) if id_new: - cursor.execute('INSERT INTO "' + cls._table + '" ' - '(id' + upd0 + ') ' - 'VALUES (' + str(id_new) + upd1 + ')', - tuple(upd2)) + insert_columns.append(table.id) + insert_values.append(id_new) + cursor.execute(*table.insert(insert_columns, + [insert_values])) else: - cursor.execute('INSERT INTO "' + cls._table + '" ' - '(' + upd0[1:] + ') ' - 'VALUES (' + upd1[1:] + ')', - tuple(upd2)) + cursor.execute(*table.insert(insert_columns, + [insert_values])) id_new = cursor.lastid() new_ids.append(id_new) except DatabaseIntegrityError, exception: with contextlib.nested(Transaction().new_cursor(), Transaction().set_user(0)): - for field_name in cls._fields: - field = cls._fields[field_name] - # Check required fields - if (field.required - and not hasattr(field, 'set') - and field_name not in ( - 'create_uid', 'create_date')): - if values.get(field_name) is None: - cls.raise_user_error('required_field', - error_args=cls._get_error_args( - field_name)) - if isinstance(field, fields.Many2One) \ - and values.get(field_name): - Model = pool.get(field.model_name) - create_records = Transaction().create_records.get( - field.model_name, set()) - delete_records = Transaction().delete_records.get( - field.model_name, set()) - if not ((Model.search([ - ('id', '=', - values[field_name]), - ], order=[]) - or values[field_name] in - create_records) - and values[field_name] not in - delete_records): - cls.raise_user_error('foreign_model_missing', - error_args=cls._get_error_args( - field_name)) - for name, _, error in cls._sql_constraints: - if name in exception[0]: - cls.raise_user_error(error) - for name, error in cls._sql_error_messages.iteritems(): - if name in exception[0]: - cls.raise_user_error(error) + cls.__raise_integrity_error(exception, values) raise - domain1, domain2 = pool.get('ir.rule').domain_get(cls.__name__, + domain = pool.get('ir.rule').domain_get(cls.__name__, mode='create') - if domain1: - in_max = Transaction().cursor.IN_MAX + if domain: for i in range(0, len(new_ids), in_max): sub_ids = new_ids[i:i + in_max] - red_sql, red_ids = reduce_ids('id', sub_ids) + red_sql = reduce_ids(table.id, sub_ids) - cursor.execute('SELECT id FROM "' + cls._table + '" WHERE ' + - red_sql + ' AND (' + domain1 + ')', red_ids + domain2) + cursor.execute(*table.select(table.id, + where=red_sql & table.id.in_(domain))) if len(cursor.fetchall()) != len(sub_ids): cls.raise_user_error('access_error', cls.__name__) - create_records = Transaction().create_records.setdefault(cls.__name__, + transaction.create_records.setdefault(cls.__name__, set()).update(new_ids) translation_values = {} @@ -355,8 +381,8 @@ class ModelSQL(ModelStorage): for fname, value in values.iteritems(): field = cls._fields[fname] if getattr(field, 'translate', False): - translation_values.setdefault(cls.__name__ + ',' + fname, - {})[new_id] = value + translation_values.setdefault( + '%s,%s' % (cls.__name__, fname), {})[new_id] = value if hasattr(field, 'set'): field.set([new_id], cls, fname, value) @@ -365,18 +391,7 @@ class ModelSQL(ModelStorage): Translation.set_ids(name, 'model', Transaction().language, translations.keys(), translations.values()) - if cls._history: - columns = ['"' + str(x) + '"' for x in cls._fields - if not hasattr(cls._fields[x], 'set')] - columns = ','.join(columns) - for i in range(0, len(new_ids), cursor.IN_MAX): - sub_ids = new_ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) - cursor.execute('INSERT INTO "' + cls._table + '__history" ' - '(' + columns + ') ' - 'SELECT ' + columns + ' ' - 'FROM "' + cls._table + '" ' - 'WHERE ' + red_sql, red_ids) + cls.__insert_history(new_ids) records = cls.browse(new_ids) cls._validate(records) @@ -402,7 +417,7 @@ class ModelSQL(ModelStorage): return [] # construct a clause for the rules : - domain1, domain2 = Rule.domain_get(cls.__name__, mode='read') + domain = Rule.domain_get(cls.__name__, mode='read') fields_related = {} datetime_fields = [] @@ -418,228 +433,209 @@ class ModelSQL(ModelStorage): if hasattr(field, 'datetime_field') and field.datetime_field: datetime_fields.append(field.datetime_field) - fields_pre = [x - for x in fields_names + fields_related.keys() + datetime_fields - if (x in cls._fields and not hasattr(cls._fields[x], 'set')) - or (x == '_timestamp')] - - res = [] - table_query = '' - table_args = [] - - if cls.table_query(): - table_query, table_args = cls.table_query() - table_query = '(' + table_query + ') AS ' + result = [] + table = cls.__table__() + table_query = cls.table_query() in_max = cursor.IN_MAX - history_order = '' - history_clause = '' - history_limit = '' - history_args = [] + history_order = None + history_clause = None + history_limit = None if (cls._history and Transaction().context.get('_datetime') and not table_query): in_max = 1 - table_query = '"' + cls._table + '__history" AS ' - history_clause = ' AND (COALESCE(write_date, create_date) <= %s)' - history_order = ' ORDER BY COALESCE(write_date, create_date) DESC' - history_limit = cursor.limit_clause('', 1) - history_args = [Transaction().context['_datetime']] - if len(fields_pre): - fields_pre2 = ['"%s"."%s" AS "%s"' % (cls._table, x, x) - for x in fields_pre if x != '_timestamp'] - if '_timestamp' in fields_pre: - if not cls.table_query(): - fields_pre2 += ['CAST(EXTRACT(EPOCH FROM ' - '(COALESCE("%s".write_date, "%s".create_date))) ' - 'AS VARCHAR) AS _timestamp' % - (cls._table, cls._table)] - if 'id' not in fields_pre: - fields_pre2 += ['"%s".id AS id' % cls._table] + table = cls.__table_history__() + column = Coalesce(table.write_date, table.create_date) + history_clause = (column <= Transaction().context['_datetime']) + history_order = column.desc + history_limit = 1 + + columns = [] + for f in fields_names + fields_related.keys() + datetime_fields: + if (f in cls._fields and not hasattr(cls._fields[f], 'set')): + columns.append(Column(table, f).as_(f)) + elif f == '_timestamp' and not table_query: + sql_type = fields.Char('timestamp').sql_type().base + columns.append(Extract('EPOCH', + Coalesce(table.write_date, table.create_date) + ).cast(sql_type).as_('_timestamp')) + + if len(columns): + if 'id' not in fields_names: + columns.append(table.id.as_('id')) for i in range(0, len(ids), in_max): sub_ids = ids[i:i + in_max] - red_sql, red_ids = reduce_ids('id', sub_ids) - if domain1: - cursor.execute('SELECT ' + ','.join(fields_pre2) - + ' FROM ' + table_query + '\"' + cls._table + '\" ' - 'WHERE ' + red_sql + history_clause + - ' AND (' + domain1 + ') ' - + history_order - + history_limit, - table_args + red_ids + history_args + domain2) - else: - cursor.execute('SELECT ' + ','.join(fields_pre2) - + ' FROM ' + table_query + '\"' + cls._table + '\" ' - 'WHERE ' + red_sql + history_clause - + history_order - + history_limit, - table_args + red_ids + history_args) + red_sql = reduce_ids(table.id, sub_ids) + where = red_sql + if history_clause: + where &= history_clause + if domain: + where &= table.id.in_(domain) + cursor.execute(*table.select(*columns, where=where, + order_by=history_order, limit=history_limit)) dictfetchall = cursor.dictfetchall() if not len(dictfetchall) == len({}.fromkeys(sub_ids)): - if domain1: - cursor.execute('SELECT id ' - 'FROM ' + table_query + '\"' + cls._table + '\" ' - 'WHERE ' + red_sql + history_clause - + history_order - + history_limit, - table_args + red_ids + history_args) + if domain: + where = red_sql + if history_clause: + where &= history_clause + where &= table.id.in_(domain) + cursor.execute(*table.select(table.id, where=where, + order_by=history_order, limit=history_limit)) rowcount = cursor.rowcount if rowcount == -1 or rowcount is None: rowcount = len(cursor.fetchall()) if rowcount == len({}.fromkeys(sub_ids)): cls.raise_user_error('access_error', cls.__name__) cls.raise_user_error('read_error', cls.__name__) - res.extend(dictfetchall) + result.extend(dictfetchall) else: - res = [{'id': x} for x in ids] + result = [{'id': x} for x in ids] - for field in fields_pre: + for column in columns: + field = column.output_name if field == '_timestamp': continue if getattr(cls._fields[field], 'translate', False): - ids = [x['id'] for x in res] - res_trans = Translation.get_ids(cls.__name__ + ',' + field, + translations = Translation.get_ids(cls.__name__ + ',' + field, 'model', Transaction().language, ids) - for i in res: - i[field] = res_trans.get(i['id'], False) or i[field] - - ids = [x['id'] for x in res] + for row in result: + row[field] = translations.get(row['id']) or row[field] # all fields for which there is a get attribute - fields_post = [x for x in + getter_fields = [f for f in fields_names + fields_related.keys() + datetime_fields - if x in cls._fields and hasattr(cls._fields[x], 'get')] + if f in cls._fields and hasattr(cls._fields[f], 'get')] func_fields = {} - for field in fields_post: - if isinstance(cls._fields[field], fields.Function): - key = (cls._fields[field].getter, - getattr(cls._fields[field], 'datetime_field', None)) + for fname in getter_fields: + field = cls._fields[fname] + if isinstance(field, fields.Function): + key = (field.getter, getattr(field, 'datetime_field', None)) func_fields.setdefault(key, []) - func_fields[key].append(field) - continue - if hasattr(cls._fields[field], 'datetime_field') \ - and cls._fields[field].datetime_field: - for record in res: - with Transaction().set_context(_datetime=record[ - cls._fields[field].datetime_field]): - res2 = cls._fields[field].get([record['id']], cls, - field, values=[record]) - record[field] = res2[record['id']] - continue - # get the value of that field for all records/ids - res2 = cls._fields[field].get(ids, cls, field, values=res) - for record in res: - record[field] = res2[record['id']] + func_fields[key].append(fname) + elif getattr(field, 'datetime_field', None): + for row in result: + with Transaction().set_context( + _datetime=row[field.datetime_field]): + date_result = field.get([row['id']], cls, fname, + values=[row]) + row[field] = date_result[row['id']] + else: + # get the value of that field for all records/ids + getter_result = field.get(ids, cls, fname, values=result) + for row in result: + row[fname] = getter_result[row['id']] + for key in func_fields: field_list = func_fields[key] - field = field_list[0] + fname = field_list[0] + field = cls._fields[fname] _, datetime_field = key if datetime_field: - for record in res: + for row in result: with Transaction().set_context( - _datetime=record[datetime_field]): - res2 = cls._fields[field].get([record['id']], cls, - field_list, values=[record]) - for field in res2: - record[field] = res2[field][record['id']] - continue - res2 = cls._fields[field].get(ids, cls, field_list, values=res) - for field in res2: - for record in res: - record[field] = res2[field][record['id']] + _datetime=row[datetime_field]): + date_results = field.get([row['id']], cls, field_list, + values=[row]) + for fname, date_result in date_results.iteritems(): + row[fname] = date_result[row['id']] + else: + getter_results = field.get(ids, cls, field_list, values=result) + for fname, getter_result in getter_results.iteritems(): + for row in result: + row[fname] = getter_result[row['id']] to_del = set() fields_related2values = {} - for field in fields_related.keys() + datetime_fields: - if field not in fields_names: - to_del.add(field) - if field not in cls._fields: + for fname in fields_related.keys() + datetime_fields: + if fname not in fields_names: + to_del.add(fname) + if fname not in cls._fields: continue - if field not in fields_related.keys(): + if fname not in fields_related: continue - fields_related2values.setdefault(field, {}) - if cls._fields[field]._type in ('many2one', 'one2one'): - if hasattr(cls._fields[field], 'model_name'): - obj = pool.get(cls._fields[field].model_name) + fields_related2values.setdefault(fname, {}) + field = cls._fields[fname] + if field._type in ('many2one', 'one2one'): + if hasattr(field, 'model_name'): + Target = pool.get(field.model_name) else: - obj = cls._fields[field].get_target() - if hasattr(cls._fields[field], 'datetime_field') \ - and cls._fields[field].datetime_field: - for record in res: - if record[field] is None: + Target = field.get_target() + if getattr(field, 'datetime_field', None): + for row in result: + if row[fname] is None: continue - with Transaction().set_context(_datetime=record[ - cls._fields[field].datetime_field]): - record2, = obj.read([record[field]], - fields_related[field]) - record_id = record2['id'] - del record2['id'] - fields_related2values[field].setdefault(record_id, {}) + with Transaction().set_context( + _datetime=row[field.datetime_field]): + date_target, = Target.read([row[fname]], + fields_related[fname]) + target_id = date_target.pop('id') + fields_related2values[fname].setdefault(target_id, {}) fields_related2values[ - field][record_id][record['id']] = record2 + fname][target_id][row['id']] = date_target else: - for record in obj.read([x[field] for x in res - if x[field]], fields_related[field]): - record_id = record['id'] - del record['id'] - fields_related2values[field].setdefault(record_id, {}) - for record2 in res: + for target in Target.read( + [r[fname] for r in result if r[fname]], + fields_related[fname]): + target_id = target.pop('id') + fields_related2values[fname].setdefault(target_id, {}) + for row in result: fields_related2values[ - field][record_id][record2['id']] = record - elif cls._fields[field]._type == 'reference': - for record in res: - if not record[field]: + fname][target_id][row['id']] = target + elif field._type == 'reference': + for row in result: + if not row[fname]: continue - model_name, record_id = record[field].split(',', 1) + model_name, record_id = row[fname].split(',', 1) if not model_name: continue record_id = int(record_id) if record_id < 0: continue - obj = pool.get(model_name) - record2, = obj.read([record_id], fields_related[field]) - del record2['id'] - fields_related2values[field][record[field]] = record2 + Target = pool.get(model_name) + target, = Target.read([record_id], fields_related[fname]) + del target['id'] + fields_related2values[fname][row[fname]] = target - if to_del or fields_related.keys() or datetime_fields: - for record in res: - for field in fields_related.keys(): - if field not in cls._fields: + if to_del or fields_related or datetime_fields: + for row in result: + for fname in fields_related: + if fname not in cls._fields: continue - for related in fields_related[field]: - if cls._fields[field]._type in ( - 'many2one', 'one2one'): - if record[field]: - record[field + '.' + related] = ( - fields_related2values[field][ - record[field]][record['id']][related]) - else: - record[field + '.' + related] = False - elif cls._fields[field]._type == 'reference': - if record[field]: - model_name, record_id = record[field].split( - ',', 1) - if not model_name: - continue - record_id = int(record_id) - if record_id < 0: - continue - record[field + '.' + related] = \ - fields_related2values[field][ - record[field]][related] + field = cls._fields[fname] + for related in fields_related[fname]: + related_name = '%s.%s' % (fname, related) + value = None + if row[fname]: + if field._type in ('many2one', 'one2one'): + value = fields_related2values[fname][ + row[fname]][row['id']][related] + elif field._type == 'reference': + model_name, record_id = row[fname + ].split(',', 1) + if model_name: + record_id = int(record_id) + if record_id >= 0: + value = fields_related2values[fname][ + row[fname]][related] + row[related_name] = value for field in to_del: - del record[field] + del row[field] - return res + return result @classmethod def write(cls, records, values): - cursor = Transaction().cursor + DatabaseIntegrityError = backend.get('DatabaseIntegrityError') + transaction = Transaction() + cursor = transaction.cursor pool = Pool() Translation = pool.get('ir.translation') Config = pool.get('ir.configuration') ids = map(int, records) + in_max = cursor.IN_MAX # Call before cursor cache cleaning trigger_eligibles = cls.trigger_write_get_eligibles(records) @@ -647,100 +643,58 @@ class ModelSQL(ModelStorage): super(ModelSQL, cls).write(records, values) if not records: - return True + return if cls.table_query(): - return True + return + table = cls.__table__() values = values.copy() # _update_tree works if only one record has changed update_tree = False - for k in cls._fields: - field = cls._fields[k] - if isinstance(field, fields.Many2One) \ - and field.model_name == cls.__name__ \ - and field.left and field.right: + for field in cls._fields.itervalues(): + if (isinstance(field, fields.Many2One) + and field.model_name == cls.__name__ + and field.left and field.right): update_tree = True if update_tree and len(records) > 1: for record in records: cls.write([record], values) - return True + return - if Transaction().timestamp: - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - clause = ('(id = %s AND ' - 'CAST(EXTRACT(EPOCH FROM ' - 'COALESCE(write_date, create_date)) AS ' + - FIELDS['numeric'].sql_type(cls.create_date)[1] + - ') > %s)') - args = [] - for i in sub_ids: - if Transaction().timestamp.get( - cls.__name__ + ',' + str(i)): - args.append(i) - args.append(Decimal(Transaction().timestamp[ - cls.__name__ + ',' + str(i)])) - if args: - cursor.execute("SELECT id " - 'FROM "' + cls._table + '" ' - 'WHERE ' + ' OR '.join( - (clause,) * (len(args) // 2)), args) - if cursor.fetchone(): - raise ConcurrencyException( - 'Records were modified in the meanwhile') - for i in ids: - if Transaction().timestamp.get(cls.__name__ + ',' + str(i)): - del Transaction().timestamp[cls.__name__ + ',' + str(i)] + cls.__check_timestamp(ids) # Clean values - for key in ('create_uid', 'create_date', 'write_uid', 'write_date', - 'id'): + for key in ('create_uid', 'create_date', + 'write_uid', 'write_date', 'id'): if key in values: del values[key] - upd0 = [] - upd1 = [] - upd_todo = [] - direct = [] - for field in values: - if not hasattr(cls._fields[field], 'set'): - if ((not getattr(cls._fields[field], 'translate', False)) - or (Transaction().language == Config.get_language())): - upd0.append(('"' + field + '"', '%s')) - upd1.append(FIELDS[cls._fields[field]._type] - .sql_format(values[field])) - direct.append(field) - else: - upd_todo.append(field) + columns = [table.write_uid, table.write_date] + update_values = [transaction.user, Now()] + store_translation = Transaction().language == Config.get_language() + for fname, value in values.iteritems(): + field = cls._fields[fname] + if not hasattr(field, 'set'): + if not getattr(field, 'translate', False) or store_translation: + columns.append(Column(table, fname)) + update_values.append(field.sql_format(value)) - upd0.append(('write_uid', '%s')) - upd0.append(('write_date', '%s')) - upd1.append(Transaction().user) - upd1.append(datetime.datetime.now()) - - domain1, domain2 = pool.get('ir.rule').domain_get(cls.__name__, - mode='write') - if domain1: - domain1 = ' AND (' + domain1 + ') ' - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) - if domain1: - cursor.execute('SELECT id FROM "' + cls._table + '" ' - 'WHERE ' + red_sql + ' ' + domain1, - red_ids + domain2) - else: - cursor.execute('SELECT id FROM "' + cls._table + '" ' - 'WHERE ' + red_sql, red_ids) + domain = pool.get('ir.rule').domain_get(cls.__name__, mode='write') + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + red_sql = reduce_ids(table.id, sub_ids) + where = red_sql + if domain: + where &= table.id.in_(domain) + cursor.execute(*table.select(table.id, where=where)) rowcount = cursor.rowcount if rowcount == -1 or rowcount is None: rowcount = len(cursor.fetchall()) if not rowcount == len({}.fromkeys(sub_ids)): - if domain1: - cursor.execute('SELECT id FROM "' + cls._table + '" ' - 'WHERE ' + red_sql, red_ids) + if domain: + cursor.execute(*table.select(table.id, where=red_sql)) rowcount = cursor.rowcount if rowcount == -1 or rowcount is None: rowcount = len(cursor.fetchall()) @@ -748,73 +702,25 @@ class ModelSQL(ModelStorage): cls.raise_user_error('access_error', cls.__name__) cls.raise_user_error('write_error', cls.__name__) try: - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET ' + ','.join([x[0] + ' = ' + x[1] for x in upd0]) - + ' WHERE ' + red_sql, upd1 + red_ids) + cursor.execute(*table.update(columns, update_values, + where=red_sql)) except DatabaseIntegrityError, exception: with contextlib.nested(Transaction().new_cursor(), Transaction().set_user(0)): - for field_name in values: - if field_name not in cls._fields: - continue - field = cls._fields[field_name] - # Check required fields - if field.required and \ - not hasattr(field, 'set') and \ - not isinstance(field, (fields.Integer, - fields.Float)) and \ - field_name not in ('create_uid', - 'create_date'): - if not values[field_name]: - cls.raise_user_error('required_field', - error_args=cls._get_error_args( - field_name)) - if isinstance(field, fields.Many2One) \ - and values[field_name]: - Model = pool.get(field.model_name) - create_records = Transaction().create_records.get( - field.model_name, set()) - delete_records = Transaction().delete_records.get( - field.model_name, set()) - if not ((Model.search([ - ('id', '=', - values[field_name]), - ], order=[]) - or (values[field_name] - in create_records)) - and (values[field_name] - not in delete_records)): - cls.raise_user_error('foreign_model_missing', - error_args=cls._get_error_args( - field_name)) - for name, _, error in cls._sql_constraints: - if name in exception[0]: - cls.raise_user_error(error) - for name, error in cls._sql_error_messages.iteritems(): - if name in exception[0]: - cls.raise_user_error(error) + cls.__raise_integrity_error(exception, values, + values.keys()) raise - for field in direct: - if getattr(cls._fields[field], 'translate', False): + for fname, value in values.iteritems(): + field = cls._fields[fname] + if getattr(field, 'translate', False): Translation.set_ids( - cls.__name__ + ',' + field, 'model', - Transaction().language, ids, [values[field]] * len(ids)) - - # call the 'set' method of fields - for field in upd_todo: - cls._fields[field].set(ids, cls, field, values[field]) - - if cls._history: - columns = ['"' + str(x) + '"' for x in cls._fields - if not hasattr(cls._fields[x], 'set')] - for obj_id in ids: - cursor.execute('INSERT INTO "' + cls._table + '__history" ' - '(' + ','.join(columns) + ') ' - 'SELECT ' + ','.join(columns) + ' ' + - 'FROM "' + cls._table + '" ' - 'WHERE id = %s', (obj_id,)) + '%s,%s' % (cls.__name__, fname), 'model', + transaction.language, ids, [value] * len(ids)) + if hasattr(field, 'set'): + field.set(ids, cls, fname, value) + cls.__insert_history(ids) cls._validate(records) field_names = cls._fields.keys() @@ -822,77 +728,54 @@ class ModelSQL(ModelStorage): cls.trigger_write(trigger_eligibles) - return True - @classmethod def delete(cls, records): - cursor = Transaction().cursor + DatabaseIntegrityError = backend.get('DatabaseIntegrityError') + transaction = Transaction() + cursor = transaction.cursor pool = Pool() Translation = pool.get('ir.translation') ids = map(int, records) + in_max = cursor.IN_MAX if not ids: - return True + return if cls.table_query(): - return True + return + table = cls.__table__() - if Transaction().delete and Transaction().delete.get(cls.__name__): + if transaction.delete and transaction.delete.get(cls.__name__): ids = ids[:] - for del_id in Transaction().delete[cls.__name__]: + for del_id in transaction.delete[cls.__name__]: for i in range(ids.count(del_id)): ids.remove(del_id) - if Transaction().timestamp: - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - clause = ('(id = %s AND ' - 'CAST(EXTRACT(EPOCH FROM ' - 'COALESCE(write_date, create_date)) AS ' + - FIELDS['numeric'].sql_type(cls.create_date)[1] + - ') > %s)') - args = [] - for i in sub_ids: - if Transaction().timestamp.get( - cls.__name__ + ',' + str(i)): - args.append(i) - args.append(Transaction().timestamp[ - cls.__name__ + ',' + str(i)]) - if args: - cursor.execute("SELECT id " - 'FROM "' + cls._table + '" ' - 'WHERE ' + ' OR '.join( - (clause,) * (len(args) / 2)), args) - if cursor.fetchone(): - raise ConcurrencyException( - 'Records were modified in the meanwhile') - for i in ids: - if Transaction().timestamp.get(cls.__name__ + ',' + str(i)): - del Transaction().timestamp[cls.__name__ + ',' + str(i)] + cls.__check_timestamp(ids) tree_ids = {} - for k in cls._fields: - field = cls._fields[k] - if isinstance(field, fields.Many2One) \ - and field.model_name == cls.__name__ \ - and field.left and field.right: - red_sql, red_ids = reduce_ids('"' + k + '"', ids) - cursor.execute('SELECT id FROM "' + cls._table + '" ' - 'WHERE ' + red_sql, red_ids) - tree_ids[k] = [x[0] for x in cursor.fetchall()] + for fname, field in cls._fields.iteritems(): + if (isinstance(field, fields.Many2One) + and field.model_name == cls.__name__ + and field.left and field.right): + tree_ids[fname] = [] + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + where = reduce_ids(Column(table, fname), sub_ids) + cursor.execute(*table.select(table.id, where=where)) + tree_ids[fname] += [x[0] for x in cursor.fetchall()] foreign_keys_tocheck = [] foreign_keys_toupdate = [] foreign_keys_todelete = [] for _, model in pool.iterobject(): - if hasattr(model, 'table_query') \ - and model.table_query(): + if hasattr(model, 'table_query') and model.table_query(): continue if not issubclass(model, ModelStorage): continue for field_name, field in model._fields.iteritems(): - if isinstance(field, fields.Many2One) \ - and field.model_name == cls.__name__: + if (isinstance(field, fields.Many2One) + and field.model_name == cls.__name__): if field.ondelete == 'CASCADE': foreign_keys_todelete.append((model, field_name)) elif field.ondelete == 'SET NULL': @@ -903,45 +786,41 @@ class ModelSQL(ModelStorage): else: foreign_keys_tocheck.append((model, field_name)) - Transaction().delete.setdefault(cls.__name__, set()).update(ids) + transaction.delete.setdefault(cls.__name__, set()).update(ids) - domain1, domain2 = pool.get('ir.rule').domain_get(cls.__name__, - mode='delete') - if domain1: - domain1 = ' AND (' + domain1 + ') ' + domain = pool.get('ir.rule').domain_get(cls.__name__, mode='delete') - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) - if domain1: - cursor.execute('SELECT id FROM "' + cls._table + '" ' - 'WHERE ' + red_sql + ' ' + domain1, - red_ids + domain2) + if domain: + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + red_sql = reduce_ids(table.id, sub_ids) + cursor.execute(*table.select(table.id, + where=red_sql & table.id.in_(domain))) rowcount = cursor.rowcount if rowcount == -1 or rowcount is None: rowcount = len(cursor.fetchall()) if not rowcount == len({}.fromkeys(sub_ids)): - cls.raise_user_error('access_error', - cls._get_name()) + cls.raise_user_error('access_error', cls._get_name()) cls.trigger_delete(records) - for i in range(0, len(ids), cursor.IN_MAX): - sub_ids = ids[i:i + cursor.IN_MAX] - sub_records = records[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + sub_records = records[i:i + in_max] + red_sql = reduce_ids(table.id, sub_ids) - Transaction().delete_records.setdefault(cls.__name__, - set()).update(sub_ids) + transaction.delete_records.setdefault(cls.__name__, + set()).update(sub_ids) for Model, field_name in foreign_keys_toupdate: if (not hasattr(Model, 'search') or not hasattr(Model, 'write')): continue - red_sql2, red_ids2 = reduce_ids('"' + field_name + '"', - sub_ids) - cursor.execute('SELECT id FROM "' + Model._table + '" ' - 'WHERE ' + red_sql2, red_ids2) + foreign_table = Model.__table__() + foreign_red_sql = reduce_ids( + Column(foreign_table, field_name), sub_ids) + cursor.execute(*foreign_table.select(foreign_table.id, + where=foreign_red_sql)) models = Model.browse([x[0] for x in cursor.fetchall()]) if models: Model.write(models, { @@ -952,10 +831,11 @@ class ModelSQL(ModelStorage): if (not hasattr(Model, 'search') or not hasattr(Model, 'delete')): continue - red_sql2, red_ids2 = reduce_ids('"' + field_name + '"', - sub_ids) - cursor.execute('SELECT id FROM "' + Model._table + '" ' - 'WHERE ' + red_sql2, red_ids2) + foreign_table = Model.__table__() + foreign_red_sql = reduce_ids( + Column(foreign_table, field_name), sub_ids) + cursor.execute(*foreign_table.select(foreign_table.id, + where=foreign_red_sql)) models = Model.browse([x[0] for x in cursor.fetchall()]) if models: Model.delete(models) @@ -975,109 +855,105 @@ class ModelSQL(ModelStorage): super(ModelSQL, cls).delete(sub_records) try: - cursor.execute('DELETE FROM "' + cls._table + '" ' - 'WHERE ' + red_sql, red_ids) + cursor.execute(*table.delete(where=red_sql)) except DatabaseIntegrityError, exception: with Transaction().new_cursor(): - for name, _, error in cls._sql_constraints: - if name in exception[0]: - cls.raise_user_error(error) - for name, error in cls._sql_error_messages.iteritems(): - if name in exception[0]: - cls.raise_user_error(error) + cls.__raise_integrity_error(exception, []) raise Translation.delete_ids(cls.__name__, 'model', ids) - if cls._history: - for obj_id in ids: - cursor.execute('INSERT INTO "' + cls._table + '__history" ' - '(id, write_uid, write_date) VALUES (%s, %s, %s)', - (obj_id, Transaction().user, datetime.datetime.now())) + cls.__insert_history(ids, deleted=True) cls._update_mptt(tree_ids.keys(), tree_ids.values()) @classmethod def search(cls, domain, offset=0, limit=None, order=None, count=False, - query_string=False): + query=False): pool = Pool() Rule = pool.get('ir.rule') - cursor = Transaction().cursor + transaction = Transaction() + cursor = transaction.cursor + in_max = cursor.IN_MAX # Get domain clauses - qu1, qu2, tables, tables_args = cls.search_domain(domain) + tables, expression = cls.search_domain(domain) # Get order by order_by = [] + order_types = { + 'DESC': Desc, + 'ASC': Asc, + } if order is None or order is False: order = cls._order - for field, otype in order: - if otype.upper() not in ('DESC', 'ASC'): - raise Exception('Error', 'Wrong order type (%s)!' % otype) - order_by2, tables2, tables2_args = cls._order_calc(field, otype) - order_by += order_by2 - for table in tables2: - if table not in tables: - tables.append(table) - if tables2_args.get(table): - tables_args.extend(tables2_args.get(table)) - - order_by = ','.join(order_by) + for fname, otype in order: + field = cls._fields[fname] + Order = order_types[otype.upper()] + forder = field.convert_order(fname, tables, cls) + order_by.extend((Order(o) for o in forder)) if type(limit) not in (float, int, long, type(None)): raise Exception('Error', 'Wrong limit type (%s)!' % type(limit)) if type(offset) not in (float, int, long, type(None)): raise Exception('Error', 'Wrong offset type (%s)!' % type(offset)) - # construct a clause for the rules : - domain1, domain2 = Rule.domain_get(cls.__name__, mode='read') - if domain1: - if qu1: - qu1 += ' AND ' + domain1 + main_table, _ = tables[None] + + def convert_from(table, tables): + right, condition = tables[None] + if table: + table = table.join(right, 'LEFT', condition) else: - qu1 = domain1 - qu2 += domain2 + table = right + for k, sub_tables in tables.iteritems(): + if k is None: + continue + table = convert_from(table, sub_tables) + return table + # Don't nested joins as SQLite doesn't support + table = convert_from(None, tables) + + # construct a clause for the rules : + domain = Rule.domain_get(cls.__name__, mode='read') + if domain: + expression &= main_table.id.in_(domain) if count: - cursor.execute(cursor.limit_clause( - 'SELECT COUNT("%s".id) FROM ' % cls._table + - ' '.join(tables) + (qu1 and ' WHERE ' + qu1 or ''), - limit, offset), tables_args + qu2) - res = cursor.fetchall() - return res[0][0] + cursor.execute(*table.select(Count(Literal(1)), + where=expression, limit=limit, offset=offset)) + return cursor.fetchone()[0] # execute the "main" query to fetch the ids we were searching for - select_fields = ['"' + cls._table + '".id AS id'] - if cls._history and Transaction().context.get('_datetime') \ - and not query_string: - select_fields += ['COALESCE("' + cls._table + '".write_date, "' - + cls._table + '".create_date) AS _datetime'] - if not query_string: - select_fields += [ - '"' + cls._table + '"."' + name + '" AS "' + name + '"' + columns = [main_table.id.as_('id')] + if (cls._history and transaction.context.get('_datetime') + and not query): + columns.append(Coalesce( + main_table.write_date, + main_table.create_date).as_('_datetime')) + if not query: + columns += [Column(main_table, name).as_(name) for name, field in cls._fields.iteritems() if not hasattr(field, 'get') and name != 'id' and not getattr(field, 'translate', False) and field.loading == 'eager'] if not cls.table_query(): - select_fields += ['CAST(EXTRACT(EPOCH FROM ' - '(COALESCE("' + cls._table + '".write_date, ' - '"' + cls._table + '".create_date))) AS VARCHAR' - ') AS _timestamp'] - query_str = cursor.limit_clause( - 'SELECT ' + ','.join(select_fields) + ' FROM ' + - ' '.join(tables) + (qu1 and ' WHERE ' + qu1 or '') + - (order_by and ' ORDER BY ' + order_by or ''), limit, offset) - if query_string: - return (query_str, tables_args + qu2) - cursor.execute(query_str, tables_args + qu2) + sql_type = fields.Char('timestamp').sql_type().base + columns += [Extract('EPOCH', + Coalesce(main_table.write_date, main_table.create_date) + ).cast(sql_type).as_('_timestamp')] + select = table.select(*columns, + where=expression, order_by=order_by, limit=limit, offset=offset) + if query: + return select + cursor.execute(*select) rows = cursor.dictfetchmany(cursor.IN_MAX) - cache = cursor.get_cache(Transaction().context) + cache = cursor.get_cache(transaction.context) if cls.__name__ not in cache: cache[cls.__name__] = LRUDict(RECORD_CACHE_SIZE) - delete_records = Transaction().delete_records.setdefault(cls.__name__, - set()) + delete_records = transaction.delete_records.setdefault(cls.__name__, + set()) keys = None for data in islice(rows, 0, cache.size_limit): if data['id'] in delete_records: @@ -1097,774 +973,88 @@ class ModelSQL(ModelStorage): cache[cls.__name__].setdefault(data['id'], {}).update(data) if len(rows) >= cursor.IN_MAX: - select_fields2 = [select_fields[0]] if (cls._history and Transaction().context.get('_datetime') - and not query_string): - select_fields2 += [select_fields[1]] - cursor.execute( - 'SELECT * FROM (' + - cursor.limit_clause( - 'SELECT ' + ','.join(select_fields2) + ' FROM ' + - ' '.join(tables) + (qu1 and ' WHERE ' + qu1 or '') + - (order_by and ' ORDER BY ' + order_by or ''), - limit, offset) + ') AS "' + cls._table + '"', - tables_args + qu2) + and not query): + columns = columns[:2] + else: + columns = columns[:1] + cursor.execute(*table.select(*columns, + where=expression, order_by=order_by, + limit=limit, offset=offset)) rows = cursor.dictfetchall() - if cls._history and Transaction().context.get('_datetime'): - res = [] + if cls._history and transaction.context.get('_datetime'): + ids = [] ids_date = {} for data in rows: if data['id'] in ids_date: if data['_datetime'] <= ids_date[data['id']]: continue - if data['id'] in res: - res.remove(data['id']) - res.append(data['id']) + if data['id'] in ids: + ids.remove(data['id']) + ids.append(data['id']) ids_date[data['id']] = data['_datetime'] to_delete = set() - for i in range(0, len(res), cursor.IN_MAX): - sub_ids = res[i:i + cursor.IN_MAX] - reduced_sql, reduced_ids = reduce_ids('id', sub_ids) - cursor.execute(('SELECT id, write_date ' - 'FROM "%s__history" WHERE %s' - 'AND (write_date IS NOT NULL AND create_date IS NULL)' - ) % (cls._table, reduced_sql), - reduced_ids) + history = cls.__table_history__() + for i in range(0, len(ids), in_max): + sub_ids = ids[i:i + in_max] + where = reduce_ids(history.id, sub_ids) + cursor.execute(*history.select(history.id, history.write_date, + where=where + & (history.write_date != None) + & (history.create_date == None))) for deleted_id, delete_date in cursor.fetchall(): if ids_date[deleted_id] < delete_date: to_delete.add(deleted_id) - return cls.browse(filter(lambda x: x not in to_delete, res)) + return cls.browse(filter(lambda x: x not in to_delete, ids)) return cls.browse([x['id'] for x in rows]) @classmethod def search_domain(cls, domain, active_test=True): ''' - Return SQL clause, tables to use and arguments for the domain + Return SQL tables and expression Set active_test to add it. ''' + transaction = Transaction() domain = cls._search_domain_active(domain, active_test=active_test) - table_query = '' - tables_args = [] - if cls.table_query(): - table_query, tables_args = cls.table_query() - table_query = '(' + table_query + ') AS ' + tables = { + None: (cls.__table__(), None) + } + if cls._history and transaction.context.get('_datetime'): + tables[None] = (cls.__table_history__(), None) - if cls._history and Transaction().context.get('_datetime'): - table_query = '"' + cls._table + '__history" AS ' + def is_leaf(expression): + return (isinstance(expression, (list, tuple)) + and len(expression) > 2 + and isinstance(expression[1], basestring) + and expression[1] in OPERATORS) # TODO remove OPERATORS test - tables = [table_query + '"' + cls._table + '"'] - - qu1, qu2 = cls.__search_domain_oper(domain, tables, tables_args) - if cls._history and Transaction().context.get('_datetime'): - if qu1: - qu1 += ' AND' - qu1 += ' (COALESCE("' + cls._table + '".write_date, "' + \ - cls._table + '".create_date) <= %s)' - qu2 += [Transaction().context['_datetime']] - return qu1, qu2, tables, tables_args - - @classmethod - def __search_domain_oper(cls, domain, tables, tables_args): - operator = 'AND' - if len(domain) and isinstance(domain[0], basestring): - if domain[0] not in ('AND', 'OR'): - raise Exception('ValidateError', 'Operator "%s" not supported' - % domain[0]) - operator = domain[0] - domain = domain[1:] - tuple_args = [] - list_args = [] - for arg in domain: - #add test for xmlrpc that doesn't handle tuple - if (isinstance(arg, tuple) - or (isinstance(arg, list) and len(arg) > 2 - and arg[1] in OPERATORS)): - tuple_args.append(tuple(arg)) - elif isinstance(arg, list): - list_args.append(arg) - - qu1, qu2 = cls.__search_domain_calc(tuple_args, tables, tables_args) - if len(qu1): - qu1 = (' ' + operator + ' ').join(qu1) - else: - qu1 = '' - - for domain2 in list_args: - qu1b, qu2b = cls.__search_domain_oper(domain2, tables, - tables_args) - if not qu1b: - qu1b = '%s' - qu2b = [True] - if qu1 and qu1b: - qu1 += ' ' + operator + ' ' + qu1b - elif qu1b: - qu1 = qu1b - qu2 += qu2b - if qu1: - qu1 = '(' + qu1 + ')' - return qu1, qu2 - - @classmethod - def __search_domain_calc(cls, domain, tables, tables_args): - pool = Pool() - domain = domain[:] - cursor = Transaction().cursor - - for arg in domain: - if arg[1] not in OPERATORS: - raise Exception('ValidateError', 'Argument "%s" not supported' - % arg[1]) - i = 0 - joins = [] - while i < len(domain): - fargs = domain[i][0].split('.', 1) - field = getattr(cls, fargs[0]) - table = cls - if len(fargs) > 1: - if field._type in ('many2one', 'reference'): - if field._type == 'many2one': - Target = pool.get(field.model_name) - m2o_search = [(fargs[1],) + tuple(domain[i][1:])] - else: - Target = pool.get(domain[i][3]) - m2o_search = [(fargs[1],) + tuple(domain[i][1:3]) - + tuple(domain[i][4:])] - if 'active' in Target._fields: - m2o_search += [('active', 'in', (True, False))] - if hasattr(field, 'search'): - conv = int if field._type == 'many2one' else str - domain.extend([(fargs[0], 'in', - map(conv, Target.search(m2o_search, - order=[])))]) - domain.pop(i) - else: - in_query = Target.search(m2o_search, order=[], - query_string=True) - if field._type == 'many2one': - domain[i] = (fargs[0], 'inselect', in_query, table) - else: - sql_type = FIELDS[ - cls.id._type].sql_type(cls.id)[0] - query = ('SELECT id FROM "' + cls._table + '" ' - 'WHERE CAST(SUBSTR("' + fargs[0] + '", ' - 'POSITION(\',\' IN "' + fargs[0] + '")' - + '+ 1) ' - 'AS ' + sql_type + ') ' - 'IN (' + in_query[0] + ') ' - 'AND "' + fargs[0] + '" ilike %s') - domain[i] = ('id', 'inselect', - (query, in_query[1] + [domain[i][3] + ',%'])) - i += 1 - continue - elif field._type in ('one2one', 'many2many', 'one2many'): - if hasattr(field, 'model_name'): - Target = pool.get(field.model_name) - else: - Target = field.get_target() - if hasattr(field, 'relation_name'): - Relation = pool.get(field.relation_name) - origin, target = field.origin, field.target - else: - Relation = Target - origin, target = field.field, 'id' - rev_field = Relation._fields[origin] - if rev_field._type == 'reference': - sql_type = FIELDS[ - cls.id._type].sql_type(cls.id)[0] - where = ' AND "' + origin + '" LIKE %s' - where_args = [cls.__name__ + ',%'] - origin = ('CAST(SUBSTR("%s", ' - 'POSITION(\',\' IN "%s") + 1) AS %s)' - % (origin, origin, sql_type)) - else: - origin = '"%s"' % origin - where = '' - where_args = [] - if hasattr(field, 'search'): - domain.extend([(fargs[0], 'in', - map(int, Target.search([ - (fargs[1], domain[i][1], - domain[i][2]), - ], order=[])))]) - domain.pop(i) - else: - query1, query2 = Target.search([ - (fargs[1], domain[i][1], domain[i][2]), - ], order=[], query_string=True) - query1 = ('SELECT %s FROM "%s" WHERE "%s" IN (%s)' % - (origin, Relation._table, target, query1)) - query1 += where - query2 += where_args - domain[i] = ('id', 'inselect', (query1, query2)) - i += 1 - continue - else: - raise Exception('ValidateError', - 'Clause on field "%s" doesn\'t work on "%s"' - % (domain[i][0], cls.__name__)) - if hasattr(field, 'search'): - clause = domain.pop(i) - domain.extend(field.search(table, clause[0], clause)) - elif field._type == 'one2many': - Field = pool.get(field.model_name) - table_query = '' - table_args = [] - if Field.table_query(): - table_query, table_args = Field.table_query() - table_query = '(' + table_query + ') AS ' - rev_field = Field._fields[field.field] - if rev_field._type == 'reference': - sql_type = FIELDS[cls.id._type].sql_type(cls.id)[0] - select = ('CAST(SUBSTR("%s", ' - 'POSITION(\',\' IN "%s") + 1) AS %s)' - % (field.field, field.field, sql_type)) - where = ' AND "' + field.field + '" LIKE %s' - where_args = [cls.__name__ + ',%'] - else: - select = '"' + field.field + '"' - where = '' - where_args = [] - - if isinstance(domain[i][2], bool) or domain[i][2] is None: - query1 = ('SELECT ' + select + ' ' - 'FROM ' + table_query + '"' + Field._table + '" ' - 'WHERE "' + field.field + '" IS NOT NULL' + where) - query2 = table_args + where_args - clause = 'inselect' - if not domain[i][2]: - clause = 'notinselect' - domain[i] = ('id', clause, (query1, query2)) - else: - if isinstance(domain[i][2], basestring): - target_field = 'rec_name' - else: - target_field = 'id' - query1, query2 = Field.search([ - (target_field, domain[i][1], domain[i][2]), - ], order=[], query_string=True) - query1 = ('SELECT ' + select + ' ' - 'FROM ' + table_query + - '"' + Field._table + '" ' - 'WHERE id IN (' + query1 + ')' + where) - query2 = table_args + query2 + where_args - domain[i] = ('id', 'inselect', (query1, query2)) - i += 1 - elif field._type in ('many2many', 'one2one'): - # XXX must find a solution for long id list - if hasattr(field, 'model_name'): - Target = pool.get(field.model_name) - else: - Target = field.get_target() - if domain[i][1] in ('child_of', 'not child_of'): - if isinstance(domain[i][2], basestring): - ids2 = map(int, Target.search([ - ('rec_name', 'ilike', domain[i][2]), - ], order=[])) - elif not isinstance(domain[i][2], list): - ids2 = [domain[i][2]] - else: - ids2 = domain[i][2] - - def _rec_get(ids, table, parent): - if not ids: - return [] - ids2 = map(int, table.search([ - (parent, 'in', ids), - (parent, '!=', None), - ], order=[])) - return ids + _rec_get(ids2, table, parent) - - if Target.__name__ != table.__name__: - if len(domain[i]) != 4: - raise Exception('Error', 'Programming error: ' - 'child_of on field "%s" is not allowed!' - % domain[i][0]) - ids2 = map(int, Target.search([ - (domain[i][3], 'child_of', ids2), - ], order=[])) - Relation = pool.get(field.relation_name) - red_sql, red_ids = reduce_ids('"' + field.target + '"', - ids2) - query1 = ('SELECT "' + field.origin + '" ' - 'FROM "' + Relation._table + '" ' - 'WHERE ' + red_sql + ' ' - 'AND "' + field.origin + '" IS NOT NULL') - query2 = red_ids - if domain[i][1] == 'child_of': - domain[i] = ('id', 'inselect', (query1, query2)) - else: - domain[i] = ('id', 'notinselect', (query1, query2)) - else: - if domain[i][1] == 'child_of': - domain[i] = ('id', 'in', ids2 + _rec_get(ids2, - table, domain[i][0])) - else: - domain[i] = ('id', 'not in', ids2 + _rec_get(ids2, - table, domain[i][0])) - else: - Relation = pool.get(field.relation_name) - table_query = '' - table_args = [] - if Relation.table_query(): - table_query, table_args = Relation.table_query() - table_query = '(' + table_query + ') AS ' - origin_field = Relation._fields[field.origin] - if origin_field._type == 'reference': - sql_type = FIELDS[cls.id._type].sql_type(cls.id)[0] - select = ('CAST(SUBSTR("%s", ' - 'POSITION(\',\' IN "%s") + 1) AS %s)' - % (field.origin, field.origin, sql_type)) - where = ' AND "' + field.origin + '" LIKE %s' - where_args = [cls.__name__ + ',%'] - else: - select = '"' + field.origin + '"' - where = '' - where_args = [] - if isinstance(domain[i][2], bool) or domain[i][2] is None: - query1 = ('SELECT ' + select + ' ' - 'FROM ' + table_query + ' ' - '"' + Relation._table + '" ' - 'WHERE "' + field.origin + '" IS NOT NULL' + where) - query2 = table_args + where_args - clause = 'inselect' - if not domain[i][2]: - clause = 'notinselect' - domain[i] = ('id', clause, (query1, query2)) - else: - if isinstance(domain[i][2], basestring): - target_field = 'rec_name' - else: - target_field = 'id' - - query1, query2 = Target.search([ - (target_field, domain[i][1], domain[i][2]), - ], order=[], query_string=True) - query1 = ('SELECT ' + select + ' ' - 'FROM ' + table_query + - '"' + Relation._table + '" ' - 'WHERE "' + field.target + '" IN (' + query1 + ')' - + where) - query2 = table_args + query2 + where_args - domain[i] = ('id', 'inselect', (query1, query2)) - i += 1 - - elif field._type == 'many2one': - # XXX must find a solution for long id list - if domain[i][1] in ('child_of', 'not child_of'): - if isinstance(domain[i][2], basestring): - Field = pool.get(field.model_name) - ids2 = map(int, Field.search([ - ('rec_name', 'like', domain[i][2]), - ], order=[])) - elif not isinstance(domain[i][2], list): - ids2 = [domain[i][2]] - else: - ids2 = domain[i][2] - - def _rec_get(ids, table, parent): - if not ids: - return [] - ids2 = map(int, table.search([ - (parent, 'in', ids), - (parent, '!=', None), - ])) - return ids + _rec_get(ids2, table, parent) - - if field.model_name != table.__name__: - if len(domain[i]) != 4: - raise Exception('Error', 'Programming error: ' - 'child_of on field "%s" is not allowed!' - % domain[i][0]) - ids2 = map(int, pool.get(field.model_name).search([ - (domain[i][3], 'child_of', ids2), - ], order=[])) - if domain[i][1] == 'child_of': - domain[i] = (domain[i][0], 'in', ids2, table) - else: - domain[i] = (domain[i][0], 'not in', ids2, table) - else: - if field.left and field.right and ids2: - red_sql, red_ids = reduce_ids('id', ids2) - cursor.execute('SELECT "' + field.left + '", ' - '"' + field.right + '" ' + - 'FROM "' + cls._table + '" ' + - 'WHERE ' + red_sql, red_ids) - clause = '(1=0) ' - for left, right in cursor.fetchall(): - clause += 'OR ' - clause += ('( "' + field.left + '" >= ' + - str(left) + ' ' + - 'AND "' + field.right + '" <= ' + - str(right) + ')') - - query = ('SELECT id FROM "' + cls._table + '" ' + - 'WHERE ' + clause) - if domain[i][1] == 'child_of': - domain[i] = ('id', 'inselect', (query, [])) - else: - domain[i] = ('id', 'notinselect', (query, [])) - else: - if domain[i][1] == 'child_of': - domain[i] = ('id', 'in', ids2 + _rec_get( - ids2, table, domain[i][0]), table) - else: - domain[i] = ('id', 'not in', ids2 + _rec_get( - ids2, table, domain[i][0]), table) - else: - if isinstance(domain[i][2], basestring): - Field = pool.get(field.model_name) - m2o_search = [('rec_name', domain[i][1], domain[i][2])] - if 'active' in Field._fields: - m2o_search += [('active', 'in', (True, False))] - res_ids = map(int, Field.search(m2o_search, order=[])) - domain[i] = (domain[i][0], 'in', res_ids, table) - else: - domain[i] += (table,) - i += 1 + def convert(domain): + if is_leaf(domain): + fname = domain[0].split('.', 1)[0] + field = cls._fields[fname] + expression = field.convert_domain(domain, tables, cls) + if not isinstance(expression, (Operator, Expression)): + return convert(expression) + return expression + elif not domain or list(domain) in (['OR'], ['AND']): + return Literal(True) + elif domain[0] == 'OR': + return Or((convert(d) for d in domain[1:])) else: - if getattr(field, 'translate', False): - if cls.__name__ == 'ir.model': - table_join = ('LEFT JOIN "ir_translation" ' - 'ON (ir_translation.name = ' - 'ir_model.model||\',%s\' ' - 'AND ir_translation.res_id IS NULL ' - 'AND ir_translation.lang = %%s ' - 'AND ir_translation.type = \'model\' ' - 'AND ir_translation.fuzzy = %%s)' - % domain[i][0]) - elif cls.__name__ == 'ir.model.field': - if domain[i][0] == 'field_description': - ttype = 'field' - else: - ttype = 'help' - table_join = ('LEFT JOIN "ir_model" ' - 'ON ir_model.id = ir_model_field.model ' - 'LEFT JOIN "ir_translation" ' - 'ON (ir_translation.name = ' - 'ir_model.model||\',\'||%s.name ' - 'AND ir_translation.res_id IS NULL ' - 'AND ir_translation.lang = %%s ' - 'AND ir_translation.type = \'%s\' ' - 'AND ir_translation.fuzzy = %%s)' - % (table._table, ttype)) - else: - table_join = ('LEFT JOIN "ir_translation" ' - 'ON (ir_translation.res_id = %s.id ' - 'AND ir_translation.name = \'%s,%s\' ' - 'AND ir_translation.lang = %%s ' - 'AND ir_translation.type = \'model\' ' - 'AND ir_translation.fuzzy = %%s)' - % (table._table, table.__name__, domain[i][0])) - table_join_args = [Transaction().language, False] + return And((convert(d) for d in ( + domain[1:] if domain[0] == 'AND' else domain))) - table_query = '' - table_args = [] - if table.table_query(): - table_query, table_args = table.table_query() - table_query = '(' + table_query + ') AS ' + expression = convert(domain) - Translation = pool.get('ir.translation') - - qu1, qu2, tables, table_args = \ - Translation.search_domain([ - ('value', domain[i][1], domain[i][2]), - ]) - qu1 = qu1.replace('"ir_translation"."value"', - 'COALESCE(NULLIF("ir_translation"."value", \'\'), ' - '"%s"."%s")' % (table._table, domain[i][0])) - query1 = ('SELECT "' + table._table + '".id ' - 'FROM ' + table_query + '"' + table._table + '" ' - + table_join + ' WHERE ' + qu1) - query2 = table_args + table_join_args + qu2 - - domain[i] = ('id', 'inselect', (query1, query2), table) - else: - domain[i] += (table,) - i += 1 - domain.extend(joins) - - qu1, qu2 = [], [] - for arg in domain: - table = cls - if len(arg) > 3: - table = arg[3] - column = getattr(table, arg[0]) - if arg[1] in ('inselect', 'notinselect'): - clause = 'IN' - if arg[1] == 'notinselect': - clause = 'NOT IN' - qu1.append('("%s"."%s" %s (%s))' % (table._table, arg[0], - clause, arg[2][0])) - qu2 += arg[2][1] - elif arg[1] in ('in', 'not in'): - if len(arg[2]) > 0: - todel = [] - if column._type != 'boolean': - for xitem in range(len(arg[2])): - if (arg[2][xitem] is False - or arg[2][xitem] is None): - todel.append(xitem) - arg2 = arg[2][:] - for xitem in todel[::-1]: - del arg2[xitem] - arg2 = [FIELDS[column._type].sql_format(x) - for x in arg2] - if len(arg2): - if reduce(lambda x, y: (x - and isinstance(y, (int, long)) - and not isinstance(y, bool)), - arg2, True): - red_sql, red_ids = reduce_ids('"%s"."%s"' - % (table._table, arg[0]), arg2) - if arg[1] == 'not in': - red_sql = '(NOT(' + red_sql + '))' - qu1.append(red_sql) - qu2 += red_ids - else: - qu1.append(('("%s"."%s" ' + arg[1] + ' (%s))') - % (table._table, arg[0], ','.join( - ('%s',) * len(arg2)))) - qu2 += arg2 - if todel: - if column._type == 'boolean': - if arg[1] == 'in': - qu1[-1] = ('(' + qu1[-1] + ' OR ' - '"%s"."%s" = %%s)' - % (table._table, arg[0])) - qu2.append(False) - else: - qu1[-1] = ('(' + qu1[-1] + ' AND ' - '"%s"."%s" != %%s)' - % (table._table, arg[0])) - qu2.append(False) - else: - if arg[1] == 'in': - qu1[-1] = ('(' + qu1[-1] + ' OR ' - '"%s"."%s" IS NULL)' - % (table._table, arg[0])) - else: - qu1[-1] = ('(' + qu1[-1] + ' AND ' - '"%s"."%s" IS NOT NULL)' - % (table._table, arg[0])) - elif todel: - if column._type == 'boolean': - if arg[1] == 'in': - qu1.append('("%s"."%s" = %%s)' - % (table._table, arg[0])) - qu2.append(False) - else: - qu1.append('("%s"."%s" != %%s)' - % (table._table, arg[0])) - qu2.append(False) - else: - if arg[1] == 'in': - qu1.append('("%s"."%s" IS NULL)' - % (table._table, arg[0])) - else: - qu1.append('("%s"."%s" IS NOT NULL)' - % (table._table, arg[0])) - else: - if arg[1] == 'in': - qu1.append(' %s') - qu2.append(False) - else: - qu1.append(' %s') - qu2.append(True) - else: - if (arg[2] is False or arg[2] is None) and (arg[1] == '='): - if column._type == 'boolean': - qu1.append('(("%s"."%s" = %%s) OR ("%s"."%s" IS NULL))' - % (table._table, arg[0], table._table, arg[0])) - qu2.append(False) - else: - qu1.append('("%s"."%s" IS NULL)' - % (table._table, arg[0])) - elif (arg[2] is False or arg[2] is None) and (arg[1] == '!='): - if column._type == 'boolean': - qu1.append('(("%s"."%s" != %%s) ' - 'AND ("%s"."%s" IS NOT NULL))' - % (table._table, arg[0], table._table, arg[0])) - qu2.append(False) - else: - qu1.append('("%s"."%s" IS NOT NULL)' - % (table._table, arg[0])) - else: - if arg[0] == 'id': - qu1.append('("%s"."%s" %s %%s)' - % (table._table, arg[0], arg[1])) - qu2.append(FIELDS[column._type] - .sql_format(arg[2])) - else: - add_null = False - if arg[1] in ('like', 'ilike'): - if not arg[2]: - qu2.append('%') - add_null = True - else: - qu2.append(FIELDS[ - column._type - ].sql_format(arg[2])) - elif arg[1] in ('not like', 'not ilike'): - if not arg[2]: - qu2.append('') - else: - qu2.append(FIELDS[ - column._type - ].sql_format(arg[2])) - add_null = True - else: - qu2.append(FIELDS[ - column._type - ].sql_format(arg[2])) - qu1.append('("%s"."%s" %s %%s)' % (table._table, - arg[0], arg[1])) - if add_null: - qu1[-1] = ('(' + qu1[-1] + ' OR ' - '"' + table._table + '"."' + arg[0] + '"' - ' IS NULL)') - - return qu1, qu2 - - @classmethod - def _order_calc(cls, field, otype): - pool = Pool() - order_by = [] - tables = [] - tables_args = {} - field_name = None - table_name = None - link_field = None - - if field in cls._fields: - table_name = cls._table - - if not hasattr(cls._fields[field], 'set'): - field_name = field - - if cls._fields[field].order_field: - field_name = cls._fields[field].order_field - - if isinstance(cls._fields[field], fields.Many2One): - obj = pool.get(cls._fields[field].model_name) - table_name = obj._table - link_field = field - field_name = None - - if obj._rec_name in obj._fields: - field_name = obj._rec_name - - if obj._order_name in obj._fields: - field_name = obj._order_name - - if not field_name: - field_name = 'id' - - order_by, tables, tables_args = obj._order_calc(field_name, - otype) - table_join = ('LEFT JOIN "' + table_name + '" AS ' - '"' + table_name + '.' + link_field + '" ON ' - '"%s.%s".id = "%s"."%s"' - % (table_name, link_field, cls._table, link_field)) - for i in range(len(order_by)): - if table_name in order_by[i]: - order_by[i] = order_by[i].replace(table_name, - table_name + '.' + link_field) - for i in range(len(tables)): - if table_name in tables[i]: - args = tables_args.pop(tables[i], []) - tables[i] = tables[i].replace(table_name, - table_name + '.' + link_field) - tables_args[tables[i]] = args - if table_join not in tables: - tables.insert(0, table_join) - return order_by, tables, tables_args - - if (field_name in cls._fields - and getattr(cls._fields[field_name], 'translate', False)): - translation_table = ('ir_translation_%s_%s' - % (table_name, field_name)) - if cls.__name__ == 'ir.model': - table_join = ('LEFT JOIN "ir_translation" ' - 'AS "%s" ON ' - '("%s".name = "ir_model".model||\',%s\' ' - 'AND "%s".res_id IS NULL ' - 'AND "%s".lang = %%s ' - 'AND "%s".type = \'model\' ' - 'AND "%s".fuzzy = %%s)' - % (translation_table, translation_table, field_name, - translation_table, translation_table, - translation_table, translation_table)) - elif cls.__name__ == 'ir.model.field': - if field_name == 'field_description': - ttype = 'field' - else: - ttype = 'help' - table_join = ('LEFT JOIN "ir_model" ON ' - 'ir_model.id = ir_model_field.model') - if table_join not in tables: - tables.append(table_join) - table_join = ('LEFT JOIN "ir_translation" ' - 'AS "%s" ON ' - '("%s".name = "ir_model".model||\',\'||%s.name ' - 'AND "%s".res_id IS NULL ' - 'AND "%s".lang = %%s ' - 'AND "%s".type = \'%s\' ' - 'AND "%s".fuzzy = %%s)' - % (translation_table, translation_table, table_name, - translation_table, translation_table, - translation_table, ttype, translation_table)) - else: - table_join = ('LEFT JOIN "ir_translation" ' - 'AS "%s" ON ' - '("%s".res_id = "%s".id ' - 'AND "%s".name = \'%s,%s\' ' - 'AND "%s".lang = %%s ' - 'AND "%s".type = \'model\' ' - 'AND "%s".fuzzy = %%s)' - % (translation_table, translation_table, table_name, - translation_table, cls.__name__, field_name, - translation_table, translation_table, - translation_table)) - if table_join not in tables: - tables.append(table_join) - tables_args[table_join] = [Transaction().language, False] - order_by.append('COALESCE(NULLIF(' - + '"' + translation_table + '".value, \'\'), ' - + '"' + table_name + '".' + field_name + ') ' + otype) - return order_by, tables, tables_args - - if (field_name in cls._fields - and cls._fields[field_name]._type == 'selection' - and cls._fields[field_name].order_field is None): - selections = cls.fields_get([field_name] - )[field_name]['selection'] - if not isinstance(selections, (tuple, list)): - selections = getattr(cls, - cls._fields[field_name].selection)() - order = 'CASE ' + table_name + '.' + field_name - for selection in selections: - order += ' WHEN \'%s\' THEN \'%s\'' % selection - order += ' ELSE ' + table_name + '.' + field_name + ' END' - order_by.append(order + ' ' + otype) - return order_by, tables, tables_args - - if field_name: - if '%(table)s' in field_name or '%(order)s' in field_name: - order_by.append(field_name % { - 'table': '"%s"' % table_name, - 'order': otype, - }) - else: - order_by.append('"%s"."%s" %s' - % (table_name, field_name, otype)) - return order_by, tables, tables_args - - raise Exception('Error', 'Wrong field name (%s) for %s in order!' % - (field, cls.__name__)) + if cls._history and transaction.context.get('_datetime'): + table, _ = tables[None] + expression &= (Coalesce(table.write_date, table.create_date) + <= transaction.context['_datetime']) + return tables, expression @classmethod def _update_mptt(cls, field_names, list_ids, values=None): @@ -1881,7 +1071,7 @@ class ModelSQL(ModelStorage): 'You can not update fields: "%s", "%s"' % (field.left, field.right)) if count is None: - cursor.execute('SELECT COUNT(1) FROM "' + cls._table + '"') + cursor.execute(*cls.__table__().select(Count(Literal(1)))) count, = cursor.fetchone() if len(ids) < count / 4: for id_ in ids: @@ -1897,6 +1087,7 @@ class ModelSQL(ModelStorage): Rebuild left, right value for the tree. ''' cursor = Transaction().cursor + table = cls.__table__() right = left + 1 childs = cls.search([ @@ -1909,10 +1100,10 @@ class ModelSQL(ModelStorage): field = cls._fields[parent] if parent_id: - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET "' + field.left + '" = %s, ' - '"' + field.right + '" = %s ' - 'WHERE id = %s', (left, right, parent_id)) + cursor.execute(*table.update( + [Column(table, field.left), Column(table, field.right)], + [left, right], + where=table.id == parent_id)) return right + 1 @classmethod @@ -1924,60 +1115,55 @@ class ModelSQL(ModelStorage): the number of children node ''' cursor = Transaction().cursor - cursor.execute('SELECT "' + left + '", "' + right + '", ' - '"' + field_name + '" ' - 'FROM "' + cls._table + '" ' - 'WHERE id = %s', (record_id,)) + table = cls.__table__() + left = Column(table, left) + right = Column(table, right) + field = Column(table, field_name) + cursor.execute(*table.select(left, right, field, + where=table.id == record_id)) fetchone = cursor.fetchone() if not fetchone: return old_left, old_right, parent_id = fetchone if old_left == old_right == 0: - cursor.execute('SELECT MAX("' + right + '") ' - 'FROM "' + cls._table + '" ' - 'WHERE "' + field_name + '" IS NULL') + cursor.execute(*table.select(Max(right), + where=field == None)) old_left, = cursor.fetchone() old_left += 1 old_right = old_left + 1 - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET "' + left + '" = %s, "' + right + '" = %s ' - 'WHERE id = %s', (old_left, old_right, record_id)) + cursor.execute(*table.update([left, right], + [old_left, old_right], + where=table.id == record_id)) size = old_right - old_left + 1 parent_right = 1 if parent_id: - cursor.execute('SELECT "' + right + '" ' - 'FROM "' + cls._table + '" ' - 'WHERE id = %s', (parent_id,)) + cursor.execute(*table.select(right, where=table.id == parent_id)) parent_right = cursor.fetchone()[0] else: - cursor.execute('SELECT MAX("' + right + '") ' - 'FROM "' + cls._table + '" ' - 'WHERE "' + field_name + '" IS NULL') + cursor.execute(*table.select(Max(right), where=field == None)) fetchone = cursor.fetchone() if fetchone: parent_right = fetchone[0] + 1 - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET "' + left + '" = "' + left + '" + %s ' - 'WHERE "' + left + '" >= %s', (size, parent_right)) - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET "' + right + '" = "' + right + '" + %s ' - 'WHERE "' + right + '" >= %s', (size, parent_right)) + cursor.execute(*table.update([left], [left + size], + where=left >= parent_right)) + cursor.execute(*table.update([right], [right + size], + where=right >= parent_right)) if old_left < parent_right: - params = (parent_right - old_left, - parent_right - old_left, - old_left, old_right) + left_delta = parent_right - old_left + right_delta = parent_right - old_left + left_cond = old_left + right_cond = old_right else: - params = (parent_right - old_left - size, - parent_right - old_left - size, - old_left + size, old_right + size) - cursor.execute('UPDATE "' + cls._table + '" ' - 'SET "' + left + '" = "' + left + '" + %s, ' - '"' + right + '" = "' + right + '" + %s ' - 'WHERE "' + left + '" >= %s ' - 'AND "' + right + '" <= %s', params) + left_delta = parent_right - old_left - size + right_delta = parent_right - old_left - size + left_cond = old_left + size + right_cond = old_right + size + cursor.execute(*table.update([left, right], + [left + left_delta, right + right_delta], + where=(left >= left_cond) & (right <= right_cond))) @classmethod def validate(cls, records): @@ -1987,23 +1173,25 @@ class ModelSQL(ModelStorage): return # Works only for a single transaction ids = map(int, records) + table = cls.__table__() + param = Flavor.get().param for _, sql, error in cls._sql_constraints: match = _RE_UNIQUE.match(sql) if match: sql = match.group(1) columns = sql.split(',') - sql_clause = ' AND '.join('%s = %%s' - % i for i in columns) - sql_clause = '(id != %s AND ' + sql_clause + ')' + sql_clause = ' AND '.join('%s = %s' + % (i, param) for i in columns) + sql_clause = '(id != ' + param + ' AND ' + sql_clause + ')' in_max = cursor.IN_MAX / (len(columns) + 1) for i in range(0, len(ids), in_max): sub_ids = ids[i:i + in_max] - red_sql, red_ids = reduce_ids('id', sub_ids) + red_sql = reduce_ids(table.id, sub_ids) cursor.execute('SELECT id,' + sql + ' ' 'FROM "' + cls._table + '" ' - 'WHERE ' + red_sql, red_ids) + 'WHERE ' + str(red_sql), red_sql.params) fetchall = cursor.fetchall() cursor.execute('SELECT id ' @@ -2020,11 +1208,11 @@ class ModelSQL(ModelStorage): sql = match.group(1) for i in range(0, len(ids), cursor.IN_MAX): sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) + red_sql = reduce_ids(table.id, sub_ids) cursor.execute('SELECT id ' 'FROM "' + cls._table + '" ' 'WHERE NOT (' + sql + ') ' - 'AND ' + red_sql, red_ids) + 'AND ' + str(red_sql), red_sql.params) if cursor.fetchone(): cls.raise_user_error(error) continue diff --git a/trytond/model/modelstorage.py b/trytond/model/modelstorage.py index ff02e484b5..3a3d420d92 100644 --- a/trytond/model/modelstorage.py +++ b/trytond/model/modelstorage.py @@ -39,9 +39,9 @@ class ModelStorage(Model): """ create_uid = fields.Many2One('res.user', 'Create User', readonly=True) - create_date = fields.DateTime('Create Date', readonly=True) + create_date = fields.Timestamp('Create Date', readonly=True) write_uid = fields.Many2One('res.user', 'Write User', readonly=True) - write_date = fields.DateTime('Write Date', readonly=True) + write_date = fields.Timestamp('Write Date', readonly=True) rec_name = fields.Function(fields.Char('Name'), 'get_rec_name', searcher='search_rec_name') @@ -684,11 +684,13 @@ class ModelStorage(Model): elif field_type == 'numeric': res = Decimal(value) if value else None elif field_type == 'date': - res = value and datetime.date(*time.strptime(value, - '%Y-%m-%d')[:3]) + res = (datetime.date( + *time.strptime(value, '%Y-%m-%d')[:3]) + if value else None) elif field_type == 'datetime': - res = value and datetime.datetime(*time.strptime(value, - '%Y-%m-%d %H:%M:%S')[:6]) + res = (datetime.datetime( + *time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6]) + if value else None) elif field_type == 'selection': res = get_selection(this_field_def['selection'], value) elif field_type == 'many2one': diff --git a/trytond/model/modelview.py b/trytond/model/modelview.py index 08ab33ca27..0bd0d01755 100644 --- a/trytond/model/modelview.py +++ b/trytond/model/modelview.py @@ -301,11 +301,13 @@ class ModelView(Model): raise_exception=False): return False if field._type in ('many2many', 'one2one'): - if not ModelAccess.check(field.target, mode='read', - raise_exception=False): + if (field.target + and not ModelAccess.check(field.target, mode='read', + raise_exception=False)): return False - elif not ModelAccess.check(field.relation_name, - mode='read', raise_exception=False): + elif (field.relation_name + and not ModelAccess.check(field.relation_name, + mode='read', raise_exception=False)): return False if field._type == 'reference': selection = field.selection diff --git a/trytond/modules/__init__.py b/trytond/modules/__init__.py index 44e5b8dba4..0530527af1 100644 --- a/trytond/modules/__init__.py +++ b/trytond/modules/__init__.py @@ -10,7 +10,9 @@ import imp import operator import ConfigParser from glob import iglob -import datetime + +from sql import Table +from sql.functions import Now import trytond.tools as tools from trytond.config import CONFIG @@ -18,6 +20,9 @@ from trytond.transaction import Transaction from trytond.cache import Cache import trytond.convert as convert +ir_module = Table('ir_module_module') +ir_model_data = Table('ir_model_data') + OPJ = os.path.join MODULES_PATH = os.path.abspath(os.path.dirname(__file__)) @@ -207,9 +212,8 @@ def load_module_graph(graph, pool, lang=None): cursor = Transaction().cursor modules = [x.name for x in graph] - cursor.execute('SELECT name, state FROM ir_module_module ' - 'WHERE name in (' + ','.join(('%s',) * len(modules)) + ')', - modules) + cursor.execute(*ir_module.select(ir_module.name, ir_module.state, + where=ir_module.name.in_(modules))) module2state = dict(cursor.fetchall()) for package in graph: @@ -256,17 +260,17 @@ def load_module_graph(graph, pool, lang=None): Translation = pool.get('ir.translation') Translation.translation_import(lang2, module, filename) - cursor.execute('SELECT id FROM ir_module_module ' - 'WHERE name = %s', (package.name,)) + cursor.execute(*ir_module.select(ir_module.id, + where=(ir_module.name == package.name))) try: module_id, = cursor.fetchone() - cursor.execute('UPDATE ir_module_module SET state = %s ' - 'WHERE id = %s', ('installed', module_id)) + cursor.execute(*ir_module.update([ir_module.state], + ['installed'], where=(ir_module.id == module_id))) except TypeError: - cursor.execute('INSERT INTO ir_module_module ' - '(create_uid, create_date, name, state) ' - 'VALUES (%s, %s, %s, %s)', (0, datetime.datetime.now(), - package.name, 'installed')) + cursor.execute(*ir_module.insert( + [ir_module.create_uid, ir_module.create_date, + ir_module.name, ir_module.state], + [[0, Now(), package.name, 'installed']])) module2state[package.name] = 'installed' cursor.commit() @@ -367,18 +371,19 @@ def load_modules(database_name, pool, update=False, lang=None): cursor = Transaction().cursor if update: # Migration from 2.2: workflow module removed - cursor.execute('DELETE FROM ir_module_module ' - 'WHERE name = %s', ('workflow',)) + cursor.execute(*ir_module.delete( + where=(ir_module.name == 'workflow'))) if 'all' in CONFIG['init']: - cursor.execute("SELECT name FROM ir_module_module " - "WHERE name != \'test\'") + cursor.execute(*ir_module.select(ir_module.name, + where=(ir_module.name != 'test'))) else: - cursor.execute("SELECT name FROM ir_module_module " - "WHERE state IN ('installed', 'to install', " - "'to upgrade', 'to remove')") + cursor.execute(*ir_module.select(ir_module.name, + where=ir_module.state.in_(('installed', 'to install', + 'to upgrade', 'to remove')))) else: - cursor.execute("SELECT name FROM ir_module_module " - "WHERE state IN ('installed', 'to upgrade', 'to remove')") + cursor.execute(*ir_module.select(ir_module.name, + where=ir_module.state.in_(('installed', 'to upgrade', + 'to remove')))) module_list = [name for (name,) in cursor.fetchall()] if update: for module in CONFIG['init'].keys(): @@ -396,21 +401,23 @@ def load_modules(database_name, pool, update=False, lang=None): raise if update: - cursor.execute("SELECT name FROM ir_module_module " - "WHERE state IN ('to remove')") + cursor.execute(*ir_module.select(ir_module.name, + where=(ir_module.state == 'to remove'))) fetchall = cursor.fetchall() if fetchall: for (mod_name,) in fetchall: #TODO check if ressource not updated by the user - cursor.execute('SELECT model, db_id FROM ir_model_data ' - 'WHERE module = %s ' - 'ORDER BY id DESC', (mod_name,)) + cursor.execute(*ir_model_data.select(ir_model_data.model, + ir_model_data.db_id, + where=(ir_model_data.module == mod_name), + order_by=ir_model_data.id.desc)) for rmod, rid in cursor.fetchall(): Model = pool.get(rmod) Model.delete([Model(rid)]) cursor.commit() - cursor.execute("UPDATE ir_module_module SET state = %s " - "WHERE state IN ('to remove')", ('uninstalled',)) + cursor.execute(*ir_module.update([ir_module.state], + ['uninstalled'], + where=(ir_module.state == 'to remove'))) cursor.commit() res = False diff --git a/trytond/protocols/dispatcher.py b/trytond/protocols/dispatcher.py index 899146a7fa..0c13d8ab8e 100644 --- a/trytond/protocols/dispatcher.py +++ b/trytond/protocols/dispatcher.py @@ -7,9 +7,12 @@ import time import sys import hashlib import pydoc + +from sql import Table + from trytond.pool import Pool from trytond import security -from trytond.backend import Database, DatabaseOperationalError +from trytond import backend from trytond.config import CONFIG from trytond.version import VERSION from trytond.transaction import Transaction @@ -19,9 +22,16 @@ from trytond.exceptions import UserError, UserWarning, NotLogged, \ from trytond.rpc import RPC +ir_configuration = Table('ir_configuration') +ir_lang = Table('ir_lang') +ir_module = Table('ir_module') +res_user = Table('res_user') + + def dispatch(host, port, protocol, database_name, user, session, object_type, object_name, method, *args, **kwargs): - + Database = backend.get('Database') + DatabaseOperationalError = backend.get('DatabaseOperationalError') if object_type == 'common': if method == 'login': try: @@ -207,6 +217,7 @@ def create(database_name, password, lang, admin_password): :param admin_password: the admin password :return: True if succeed ''' + Database = backend.get('Database') security.check_super(password) res = False logger = logging.getLogger('database') @@ -224,28 +235,25 @@ def create(database_name, password, lang, admin_password): with Transaction().start(database_name, 0) as transaction: database.init(transaction.cursor) - transaction.cursor.execute('INSERT INTO ir_configuration ' - '(language) VALUES (%s)', (lang,)) + transaction.cursor.execute(*ir_configuration.insert( + [ir_configuration.language], [[lang]])) transaction.cursor.commit() pool = Pool(database_name) pool.init(update=True, lang=[lang]) with Transaction().start(database_name, 0) as transaction: cursor = transaction.cursor - #XXX replace with model write - cursor.execute('UPDATE ir_lang ' - 'SET translatable = %s ' - 'WHERE code = %s', (True, lang)) - cursor.execute('UPDATE res_user ' - 'SET language = (' + - cursor.limit_clause('SELECT id FROM ir_lang ' - 'WHERE code = %s', 1) + ')' - 'WHERE login <> \'root\'', (lang,)) + cursor.execute(*ir_lang.update([ir_lang.translatable], [True], + where=(ir_lang.code == lang))) + cursor.execute(*res_user.update([res_user.language], + [ir_lang.select(ir_lang.id, + where=(ir_lang.code == lang), + limit=1)], + where=(res_user.login != 'root'))) admin_password = hashlib.sha1(admin_password.encode('utf-8'))\ .hexdigest() - cursor.execute('UPDATE res_user ' - 'SET password = %s ' - 'WHERE login = \'admin\'', (admin_password,)) + cursor.execute(*res_user.update([res_user.password], + [admin_password], where=(res_user.login == 'admin'))) Module = pool.get('ir.module.module') if Module: Module.update_list() @@ -262,6 +270,7 @@ def create(database_name, password, lang, admin_password): def drop(database_name, password): + Database = backend.get('Database') security.check_super(password) Database(database_name).close() # Sleep to let connections close @@ -288,6 +297,7 @@ def drop(database_name, password): def dump(database_name, password): + Database = backend.get('Database') security.check_super(password) Database(database_name).close() # Sleep to let connections close @@ -300,6 +310,7 @@ def dump(database_name, password): def restore(database_name, password, data, update=False): + Database = backend.get('Database') logger = logging.getLogger('database') security.check_super(password) try: @@ -313,12 +324,11 @@ def restore(database_name, password, data, update=False): logger.info('RESTORE DB: %s' % (database_name)) if update: cursor = Database(database_name).connect().cursor() - cursor.execute('SELECT code FROM ir_lang ' - 'WHERE translatable') + cursor.execute(*ir_lang.select(ir_lang.code, + where=ir_lang.translatable)) lang = [x[0] for x in cursor.fetchall()] - cursor.execute('UPDATE ir_module_module SET ' - "state = 'to upgrade' " - "WHERE state = 'installed'") + cursor.execute(*ir_module.update([ir_module.state], ['to upgrade'], + where=(ir_module.state == 'installed'))) cursor.commit() cursor.close() Pool(database_name).init(update=update, lang=lang) diff --git a/trytond/protocols/webdav.py b/trytond/protocols/webdav.py index e7ad11bfca..98b12ff00e 100644 --- a/trytond/protocols/webdav.py +++ b/trytond/protocols/webdav.py @@ -25,7 +25,7 @@ from trytond.config import CONFIG from trytond.security import login from trytond.version import PACKAGE, VERSION, WEBSITE from trytond.tools.misc import LocalDict -from trytond.backend import Database +from trytond import backend from trytond.pool import Pool from trytond.transaction import Transaction from trytond.cache import Cache @@ -157,7 +157,7 @@ class TrytonDAVInterface(iface.dav_interface): res = [] dbname, dburi = self._get_dburi(uri) if not dbname: - database = Database().connect() + database = backend.get('Database')().connect() cursor = database.cursor() try: lists = database.list(cursor) @@ -554,7 +554,7 @@ class WebDAVAuthRequestHandler(WebDAVServer.DAVRequestHandler): def get_userinfo(self, user, password, command=''): path = urlparse.urlparse(self.path).path dbname = urllib.unquote_plus(path.split('/', 2)[1]) - database = Database().connect() + database = backend.get('Database')().connect() cursor = database.cursor() databases = database.list(cursor) cursor.close() diff --git a/trytond/res/ir.py b/trytond/res/ir.py index 046f9aab04..784ca5343a 100644 --- a/trytond/res/ir.py +++ b/trytond/res/ir.py @@ -1,7 +1,7 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from ..model import ModelSQL, fields -from ..backend import TableHandler +from .. import backend from ..transaction import Transaction from ..pool import Pool, PoolMeta @@ -24,11 +24,12 @@ class UIMenuGroup(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor # Migration from 1.0 table name change TableHandler.table_rename(cursor, 'ir_ui_menu_group_rel', cls._table) TableHandler.sequence_rename(cursor, 'ir_ui_menu_group_rel_id_seq', - cls._table + '_id_seq') + cls._table + '_id_seq') # Migration from 2.0 menu_id and gid renamed into menu group table = TableHandler(cursor, cls, module_name) table.column_rename('menu_id', 'menu') @@ -65,11 +66,12 @@ class ActionGroup(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor # Migration from 1.0 table name change TableHandler.table_rename(cursor, 'ir_action_group_rel', cls._table) TableHandler.sequence_rename(cursor, 'ir_action_group_rel_id_seq', - cls._table + '_id_seq') + cls._table + '_id_seq') # Migration from 2.0 action_id and gid renamed into action and group table = TableHandler(cursor, cls, module_name) table.column_rename('action_id', 'action') @@ -115,6 +117,7 @@ class ModelFieldGroup(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor # Migration from 1.0 table name change TableHandler.table_rename(cursor, 'ir_model_field_group_rel', @@ -174,11 +177,12 @@ class RuleGroupGroup(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor # Migration from 1.0 table name change TableHandler.table_rename(cursor, 'group_rule_group_rel', cls._table) TableHandler.sequence_rename(cursor, 'group_rule_group_rel_id_seq', - cls._table + '_id_seq') + cls._table + '_id_seq') # Migration from 2.0 rule_group_id and group_id renamed into rule_group # and group table = TableHandler(cursor, cls, module_name) @@ -197,11 +201,12 @@ class RuleGroupUser(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor # Migration from 1.0 table name change TableHandler.table_rename(cursor, 'user_rule_group_rel', cls._table) TableHandler.sequence_rename(cursor, 'user_rule_group_rel_id_seq', - cls._table + '_id_seq') + cls._table + '_id_seq') # Migration from 2.0 rule_group_id and user_id renamed into rule_group # and user table = TableHandler(cursor, cls, module_name) diff --git a/trytond/res/user.py b/trytond/res/user.py index 59f16bc4a0..2cd6e6dc74 100644 --- a/trytond/res/user.py +++ b/trytond/res/user.py @@ -9,11 +9,13 @@ import time import datetime from itertools import groupby, ifilter from operator import attrgetter +from sql import Literal +from sql.aggregate import Count from ..model import ModelView, ModelSQL, fields from ..wizard import Wizard, StateView, Button, StateTransition from ..tools import safe_eval -from ..backend import TableHandler +from .. import backend from ..transaction import Transaction from ..cache import Cache from ..pool import Pool @@ -108,6 +110,7 @@ class User(ModelSQL, ModelView): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(User, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) @@ -440,9 +443,9 @@ class User(ModelSQL, ModelView): if result: return result cursor = Transaction().cursor - cursor.execute('SELECT id, password, salt ' - 'FROM "' + cls._table + '" ' - 'WHERE login = %s AND active', (login,)) + table = cls.__table__() + cursor.execute(*table.select(table.id, table.password, table.salt, + where=(table.login == login) & table.active)) result = cursor.fetchone() or (None, None, None) cls._get_login_cache.set(login, result) return result @@ -478,6 +481,7 @@ class LoginAttempt(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') super(LoginAttempt, cls).__register__(module_name) table = TableHandler(Transaction().cursor, cls, module_name) @@ -499,15 +503,16 @@ class LoginAttempt(ModelSQL): @classmethod def remove(cls, login): cursor = Transaction().cursor - cursor.execute('DELETE FROM "' + cls._table + '" WHERE "login" = %s', - (login,)) + table = cls.__table__() + cursor.execute(*table.delete(where=table.login == login)) @classmethod def count(cls, login): cursor = Transaction().cursor - cursor.execute('SELECT count(1) FROM "' + cls._table + '" ' - 'WHERE "login" = %s AND create_date >= %s', - (login, cls.delay())) + table = cls.__table__() + cursor.execute(*table.select(Count(Literal(1)), + where=(table.login == login) + & (table.create_date >= cls.delay()))) return cursor.fetchone()[0] @@ -549,11 +554,12 @@ class UserGroup(ModelSQL): @classmethod def __register__(cls, module_name): + TableHandler = backend.get('TableHandler') cursor = Transaction().cursor # Migration from 1.0 table name change TableHandler.table_rename(cursor, 'res_group_user_rel', cls._table) TableHandler.sequence_rename(cursor, 'res_group_user_rel_id_seq', - cls._table + '_id_seq') + cls._table + '_id_seq') # Migration from 2.0 uid and gid rename into user and group table = TableHandler(cursor, cls, module_name) table.column_rename('uid', 'user') diff --git a/trytond/server.py b/trytond/server.py index 2bcafdadb3..9da2381deb 100644 --- a/trytond/server.py +++ b/trytond/server.py @@ -9,13 +9,17 @@ import sys import os import signal import time -from trytond.config import CONFIG from getpass import getpass import hashlib import threading import string import random +from trytond.config import CONFIG +from trytond import backend +from trytond.pool import Pool +from trytond.monitor import monitor + class TrytonServer(object): @@ -73,9 +77,7 @@ class TrytonServer(object): def run(self): "Run the server and never return" - from trytond.backend import Database - from trytond.pool import Pool - from trytond.monitor import monitor + Database = backend.get('Database') update = bool(CONFIG['init'] or CONFIG['update']) init = {} diff --git a/trytond/test/mptt.py b/trytond/test/mptt.py index 12e02bd2d4..ee9486d1b0 100644 --- a/trytond/test/mptt.py +++ b/trytond/test/mptt.py @@ -24,6 +24,11 @@ class MPTT(ModelSQL, ModelView): super(MPTT, cls).validate(record) cls.check_recursion(record) + @staticmethod + def order_sequence(tables): + table, _ = tables[None] + return [table.sequence == None, table.sequence] + @staticmethod def default_active(): return True diff --git a/trytond/tests/test_fields.py b/trytond/tests/test_fields.py index 7206e46934..6bac0f4ef2 100644 --- a/trytond/tests/test_fields.py +++ b/trytond/tests/test_fields.py @@ -2247,11 +2247,6 @@ class FieldsTestCase(unittest.TestCase): }]) self.assert_(time7) - time8, = self.time.create([{ - 'time': False, - }]) - self.assert_(time8) - time9, = self.time.create([{ 'time': evening.replace(microsecond=1), }]) diff --git a/trytond/tests/test_mptt.py b/trytond/tests/test_mptt.py index 4b2da8f370..a15a2211d1 100644 --- a/trytond/tests/test_mptt.py +++ b/trytond/tests/test_mptt.py @@ -18,7 +18,7 @@ class MPTTTestCase(unittest.TestCase): install_module('test') self.mptt = POOL.get('test.mptt') - def CheckTree(self, parent_id=False, left=-1, right=sys.maxint): + def CheckTree(self, parent_id=None, left=-1, right=sys.maxint): childs = self.mptt.search([ ('parent', '=', parent_id), ], order=[('left', 'ASC')]) diff --git a/trytond/tests/test_sequence.py b/trytond/tests/test_sequence.py index ab0454ea34..c942506cb0 100644 --- a/trytond/tests/test_sequence.py +++ b/trytond/tests/test_sequence.py @@ -36,13 +36,13 @@ class SequenceTestCase(unittest.TestCase): self.sequence.write([sequence], { 'number_increment': 10, }) - self.assertEqual(self.sequence.get_id([sequence]), '2') - self.assertEqual(self.sequence.get_id([sequence]), '12') + self.assertEqual(self.sequence.get_id(sequence), '2') + self.assertEqual(self.sequence.get_id(sequence), '12') self.sequence.write([sequence], { 'padding': 3, }) - self.assertEqual(self.sequence.get_id([sequence]), '022') + self.assertEqual(self.sequence.get_id(sequence), '022') transaction.cursor.rollback() diff --git a/trytond/tests/test_tools.py b/trytond/tests/test_tools.py index 3a04553d93..86881afe21 100644 --- a/trytond/tests/test_tools.py +++ b/trytond/tests/test_tools.py @@ -5,6 +5,8 @@ import unittest import datetime +import sql +import sql.operators from trytond.tools import reduce_ids, safe_eval, datetime_strftime, \ reduce_domain @@ -14,62 +16,64 @@ class ToolsTestCase(unittest.TestCase): ''' Test tools. ''' + table = sql.Table('test') def test0000reduce_ids_empty(self): ''' Test reduce_ids empty list. ''' - self.assert_(('(%s)', [False]) == reduce_ids('id', [])) + self.assertEqual(reduce_ids(self.table.id, []), sql.Literal(False)) def test0010reduce_ids_continue(self): ''' Test reduce_ids continue list. ''' - self.assert_(('(((id >= %s) AND (id <= %s)))', [0, 9]) - == reduce_ids('id', range(10))) + self.assertEqual(reduce_ids(self.table.id, range(10)), + sql.operators.Or(((self.table.id >= 0) & (self.table.id <= 9),))) def test0020reduce_ids_one_hole(self): ''' Test reduce_ids continue list with one hole. ''' - self.assert_(('(((id >= %s) AND (id <= %s)) OR ' - '((id >= %s) AND (id <= %s)))', [0, 9, 20, 29]) - == reduce_ids('id', range(10) + map(lambda x: x + 20, range(10)))) + self.assertEqual(reduce_ids(self.table.id, range(10) + range(20, 30)), + ((self.table.id >= 0) & (self.table.id <= 9)) + | ((self.table.id >= 20) & (self.table.id <= 29))) def test0030reduce_ids_short_continue(self): ''' Test reduce_ids short continue list. ''' - self.assert_(('((id IN (%s,%s,%s,%s)))', [0, 1, 2, 3]) - == reduce_ids('id', range(4))) + self.assertEqual(reduce_ids(self.table.id, range(4)), + sql.operators.Or((self.table.id.in_(range(4)),))) def test0040reduce_ids_complex(self): ''' Test reduce_ids complex list. ''' - self.assertEqual(('(((id >= %s) AND (id <= %s)) OR ' - '(id IN (%s,%s,%s,%s,%s)))', [0, 14, 25, 26, 27, 28, 29]), - reduce_ids('id', range(10) + map(lambda x: x + 25, range(5)) - + map(lambda x: x + 5, range(10)))) + self.assertEqual(reduce_ids(self.table.id, + range(10) + range(25, 30) + range(15, 20)), + (((self.table.id >= 0) & (self.table.id <= 14)) + | (self.table.id.in_(range(25, 30))))) def test0050reduce_ids_complex_small_continue(self): ''' Test reduce_ids complex list with small continue. ''' - self.assertEqual(('(((id >= %s) AND (id <= %s)) ' - 'OR (id IN (%s,%s,%s,%s)))', [1, 12, 15, 18, 19, 21]), - reduce_ids('id', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 18, - 19, 21])) + self.assertEqual(reduce_ids(self.table.id, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 18, 19, 21]), + (((self.table.id >= 1) & (self.table.id <= 12)) + | (self.table.id.in_([15, 18, 19, 21])))) def test0055reduce_ids_float(self): ''' Test reduce_ids with integer as float. ''' - self.assert_(('(((id >= %s) AND (id <= %s)) OR (id IN (%s,%s,%s,%s)))', - [1.0, 12.0, 15.0, 18.0, 19.0, 21.0]) - == reduce_ids('id', [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, - 10.0, 11.0, 12.0, 15.0, 18.0, 19.0, 21.0])) - self.assertRaises(AssertionError, reduce_ids, 'id', [1.1]) + self.assertEqual(reduce_ids(self.table.id, + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + 15.0, 18.0, 19.0, 21.0]), + (((self.table.id >= 1.0) & (self.table.id <= 12.0)) + | (self.table.id.in_([15.0, 18.0, 19.0, 21.0])))) + self.assertRaises(AssertionError, reduce_ids, self.table.id, [1.1]) def test0060safe_eval_builtin(self): ''' diff --git a/trytond/tests/test_tryton.py b/trytond/tests/test_tryton.py index 576ed9ecab..e5ffc9bc4b 100644 --- a/trytond/tests/test_tryton.py +++ b/trytond/tests/test_tryton.py @@ -42,7 +42,7 @@ if not CONFIG['admin_passwd']: CONFIG['admin_passwd'] = 'admin' from trytond.pool import Pool -from trytond.backend import Database +from trytond import backend from trytond.protocols.dispatcher import create from trytond.transaction import Transaction from trytond.pyson import PYSONEncoder, Eval @@ -56,7 +56,7 @@ else: USER = 1 USER_PASSWORD = 'admin' CONTEXT = {} -DB = Database(DB_NAME) +DB = backend.get('Database')(DB_NAME) Pool.test = True POOL = Pool(DB_NAME) @@ -112,6 +112,7 @@ def install_module(name): ''' Install module for the tested database ''' + Database = backend.get('Database') database = Database().connect() cursor = database.cursor() databases = database.list(cursor) diff --git a/trytond/tools/misc.py b/trytond/tools/misc.py index a632f7c974..e22d50f6ae 100644 --- a/trytond/tools/misc.py +++ b/trytond/tools/misc.py @@ -11,6 +11,8 @@ from threading import local import smtplib import dis from decimal import Decimal +from sql import Literal +from sql.operators import Or from trytond.config import CONFIG from trytond.const import OPERATORS @@ -323,14 +325,10 @@ class LocalDict(local): def reduce_ids(field, ids): ''' - Return a small SQL clause for ids - - :param field: the field of the clause - :param ids: the list of ids - :return: sql string and sql param + Return a small SQL expression for the list of ids and the sql column ''' if not ids: - return '(%s)', [False] + return Literal(False) assert all(x.is_integer() for x in ids if isinstance(x, float)), \ 'ids must be integer' ids = map(int, ids) @@ -338,8 +336,7 @@ def reduce_ids(field, ids): prev = ids.pop(0) continue_list = [prev, prev] discontinue_list = [] - sql = [] - args = [] + sql = Or() for i in ids: if i == prev: continue @@ -348,9 +345,8 @@ def reduce_ids(field, ids): discontinue_list.extend([continue_list[0] + x for x in range(continue_list[-1] - continue_list[0] + 1)]) else: - sql.append('((' + field + ' >= %s) AND (' + field + ' <= %s))') - args.append(continue_list[0]) - args.append(continue_list[-1]) + sql.append((field >= continue_list[0]) + & (field <= continue_list[-1])) continue_list = [] continue_list.append(i) prev = i @@ -358,14 +354,10 @@ def reduce_ids(field, ids): discontinue_list.extend([continue_list[0] + x for x in range(continue_list[-1] - continue_list[0] + 1)]) else: - sql.append('((' + field + ' >= %s) AND (' + field + ' <= %s))') - args.append(continue_list[0]) - args.append(continue_list[-1]) + sql.append((field >= continue_list[0]) & (field <= continue_list[-1])) if discontinue_list: - sql.append('(' + field + ' IN (' + - ','.join(('%s',) * len(discontinue_list)) + '))') - args.extend(discontinue_list) - return '(' + ' OR '.join(sql) + ')', args + sql.append(field.in_(discontinue_list)) + return sql _ALLOWED_CODES = set(dis.opmap[x] for x in [ 'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP', 'BUILD_LIST', diff --git a/trytond/transaction.py b/trytond/transaction.py index b2c27833d5..ab7796dfee 100644 --- a/trytond/transaction.py +++ b/trytond/transaction.py @@ -1,8 +1,10 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from threading import local +from sql import Flavor + from trytond.tools.singleton import Singleton -from trytond.backend import Database +from trytond import backend class _TransactionManager(object): @@ -67,10 +69,12 @@ class Transaction(local): ''' Start transaction ''' + Database = backend.get('Database') assert self.user is None assert self.cursor is None assert self.context is None database = Database(database_name).connect() + Flavor.set(Database.flavor) self.cursor = database.cursor(readonly=readonly) self.user = user self.context = context or {} @@ -124,6 +128,7 @@ class Transaction(local): return manager def new_cursor(self): + Database = backend.get('Database') manager = _CursorManager(self.cursor) database = Database(self.cursor.database_name).connect() self.cursor = database.cursor() diff --git a/trytond/webdav/webdav.py b/trytond/webdav/webdav.py index 258955a42b..d62179dab7 100644 --- a/trytond/webdav/webdav.py +++ b/trytond/webdav/webdav.py @@ -9,6 +9,9 @@ import encodings import uuid import datetime from dateutil.relativedelta import relativedelta +from sql.functions import Extract +from sql.conditionals import Coalesce + from trytond.model import ModelView, ModelSQL, fields from trytond.tools import reduce_ids, safe_eval from trytond.transaction import Transaction @@ -45,8 +48,8 @@ class Collection(ModelSQL, ModelView): childs = fields.One2Many('webdav.collection', 'parent', 'Children') model = fields.Many2One('ir.model', 'Model') domain = fields.Char('Domain') - complete_name = fields.Function(fields.Char('Complete Name', - order_field='name'), 'get_rec_name') + complete_name = fields.Function(fields.Char('Complete Name'), + 'get_rec_name') @classmethod def __setup__(cls): @@ -66,6 +69,10 @@ class Collection(ModelSQL, ModelView): '.pdf': 'application/pdf', } + @classmethod + def order_complet_name(cls, tables): + return cls.name.convert_order('name', tables, cls) + @staticmethod def default_domain(): return '[]' @@ -424,13 +431,13 @@ class Collection(ModelSQL, ModelView): ids = [object_id] res = None cursor = Transaction().cursor + table = Model.__table__() for i in range(0, len(ids), cursor.IN_MAX): sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) - cursor.execute('SELECT id, ' - 'EXTRACT(epoch FROM create_date) ' - 'FROM "' + Model._table + '" ' - 'WHERE ' + red_sql, red_ids) + red_sql = reduce_ids(table.id, sub_ids) + cursor.execute(*table.select(table.id, + Extract('EPOCH', table.create_date), + where=red_sql)) for object_id2, date in cursor.fetchall(): if object_id2 == object_id: res = date @@ -461,14 +468,14 @@ class Collection(ModelSQL, ModelView): ids = [object_id] res = None cursor = Transaction().cursor + table = Model.__table__() for i in range(0, len(ids), cursor.IN_MAX): sub_ids = ids[i:i + cursor.IN_MAX] - red_sql, red_ids = reduce_ids('id', sub_ids) - cursor.execute('SELECT id, ' - 'EXTRACT(epoch FROM ' - 'COALESCE(write_date, create_date)) ' - 'FROM "' + Model._table + '" ' - 'WHERE ' + red_sql, red_ids) + red_sql = reduce_ids(table.id, sub_ids) + cursor.execute(*table.select(table.id, + Extract('EPOCH', + Coalesce(table.write_date, table.create_date)), + where=red_sql)) for object_id2, date in cursor.fetchall(): if object_id2 == object_id: res = date diff --git a/trytond/webdav/webdav.xml b/trytond/webdav/webdav.xml index ff92f68b23..2a6ab872f3 100644 --- a/trytond/webdav/webdav.xml +++ b/trytond/webdav/webdav.xml @@ -32,7 +32,7 @@ this repository contains the full copyright notices and license terms. --> Collections ir.action.act_window webdav.collection - [('parent', '=', False)] + [('parent', '=', None)] diff --git a/trytond/wizard/wizard.py b/trytond/wizard/wizard.py index fe25e1a4b9..a4b2cc7ea5 100644 --- a/trytond/wizard/wizard.py +++ b/trytond/wizard/wizard.py @@ -177,53 +177,8 @@ class Wizard(WarningErrorMixin, URLMixin, PoolBase): super(Wizard, cls).__register__(module_name) pool = Pool() Translation = pool.get('ir.translation') - cursor = Transaction().cursor - for state_name, state in cls.states.iteritems(): - if isinstance(state, StateView): - for button in state.buttons: - cursor.execute('SELECT id, name, src ' - 'FROM ir_translation ' - 'WHERE module = %s ' - 'AND lang = %s ' - 'AND type = %s ' - 'AND name = %s', - (module_name, 'en_US', 'wizard_button', - cls.__name__ + ',' + state_name + ',' + - button.state)) - res = cursor.dictfetchall() - src_md5 = Translation.get_src_md5(button.string) - if not res: - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, ' - 'fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (cls.__name__ + ',' + state_name + ',' + - button.state, - 'en_US', 'wizard_button', button.string, - src_md5, '', module_name, False)) - elif res[0]['src'] != button.string: - cursor.execute('UPDATE ir_translation ' - 'SET src = %s, src_md5 = %s ' - 'WHERE id = %s', - (button.string, src_md5, res[0]['id'])) - - cursor.execute('SELECT id, src FROM ir_translation ' - 'WHERE lang = %s ' - 'AND type = %s ' - 'AND name = %s', - ('en_US', 'error', cls.__name__)) - trans_error = {} - for trans in cursor.dictfetchall(): - trans_error[trans['src']] = trans - - for error in cls._error_messages.values(): - if error not in trans_error: - error_md5 = Translation.get_src_md5(error) - cursor.execute('INSERT INTO ir_translation ' - '(name, lang, type, src, src_md5, value, module, fuzzy) ' - 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)', - (cls.__name__, 'en_US', 'error', error, error_md5, '', - module_name, False)) + Translation.register_wizard(cls, module_name) + Translation.register_error_messages(cls, module_name) @classmethod def create(cls):