From ffc8d2ba57ff82643aa80cdb2230af7a01ba41ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Mon, 7 Sep 2009 10:39:09 +0200 Subject: [PATCH] Fill the cursor cache at search and use it in search_read and in one2many get --- CHANGELOG | 1 + trytond/model/fields/one2many.py | 26 ++++++++- trytond/model/modelsql.py | 94 +++++++++++++++++++++++++++----- trytond/model/modelstorage.py | 31 ++++++++++- 4 files changed, 132 insertions(+), 20 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f7ffdf3784..9bc6f1efc3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,4 @@ +* Fill the cursor cache at search * Allow rpc on today of ir.date * Use the module dependency order to apply views that inherits * Allow to update database at the end of restore diff --git a/trytond/model/fields/one2many.py b/trytond/model/fields/one2many.py index 977171d673..6b0136b7ae 100644 --- a/trytond/model/fields/one2many.py +++ b/trytond/model/fields/one2many.py @@ -49,6 +49,9 @@ class One2Many(Field): :param context: the context :return: a dictionary with ids as key and values as value ''' + if context is None: + context = {} + res = {} for i in ids: res[i] = [] @@ -59,9 +62,26 @@ class One2Many(Field): [(self.field, 'in', sub_ids)], order=self.order, context=context) - for i in model.pool.get(self.model_name).read(cursor, user, ids2, - [self.field], context=context): - res[i[self.field]].append(i['id']) + cache_ctx = context.copy() + for i in ('_timestamp', '_delete', '_create_records', + '_delete_records'): + if i in cache_ctx: + del cache_ctx[i] + cache = cursor.cache.setdefault(repr(cache_ctx), {}) + cache.setdefault(self.model_name, {}) + ids3 = [] + for i in ids2: + if i in cache[self.model_name] \ + and self.field in cache[self.model_name][i]: + res[cache[self.model_name][i][self.field].id].append(i) + else: + ids3.append(i) + + if ids3: + for i in model.pool.get(self.model_name).read(cursor, user, ids3, + [self.field], context=context): + res[i[self.field]].append(i['id']) + for val in res.values(): val.sort(lambda x, y: cmp(ids2.index(x), ids2.index(y))) return res diff --git a/trytond/model/modelsql.py b/trytond/model/modelsql.py index 0013847374..f6c78583da 100644 --- a/trytond/model/modelsql.py +++ b/trytond/model/modelsql.py @@ -5,6 +5,7 @@ from trytond.model import ModelStorage, OPERATORS from trytond.model import fields from trytond.backend import FIELDS, TableHandler from trytond.backend import DatabaseIntegrityError, Database +from trytond.model.browse import BrowseRecord, BrowseRecordNull import datetime import re _RE_UNIQUE = re.compile('UNIQUE\s*\((.*)\)', re.I) @@ -1161,34 +1162,99 @@ class ModelSQL(ModelStorage): res = cursor.fetchall() return res[0][0] # execute the "main" query to fetch the ids we were searching for - select_field = '"' + self._table + '".id' + select_fields = ['"' + self._table + '".id'] if self._history and context.get('_datetime') \ and not query_string: - select_field += ', COALESCE("' + self._table + '".write_date, "' + \ - self._table + '".create_date)' - query_str = 'SELECT ' + select_field + ' FROM ' + \ + select_fields += ['COALESCE("' + self._table + '".write_date, "' + \ + self._table + '".create_date) AS _datetime'] + if not query_string: + select_fields += [(x[0] in ('create_date', 'write_date')) \ + and ('date_trunc(\'second\', "' + \ + self._table + '"."' + x[0] + '") AS "' + x[0] + '"') \ + or '"' + self._table + '"."' + x[0] + '"' \ + for x in self._columns.iteritems() \ + if not hasattr(x[1], 'get') \ + and (x[0] != 'id') + and (not x[1].translate \ + and x[1]._type not in ('text', 'binary'))] + if not self.table_query(context): + select_fields += ['(COALESCE("' + self._table + '".write_date,'\ + '"' + self._table + '".create_date)) ' \ + 'AS _timestamp'] + else: + select_fields += ['now()::timestamp AS _timestamp'] + query_str = 'SELECT ' + ','.join(select_fields) + ' FROM ' + \ ' '.join(tables) + (qu1 and ' WHERE ' + qu1 or '') + \ (order_by and ' ORDER BY ' + order_by or '') + \ limit_str + offset_str if query_string: return (query_str, tables_args + qu2) - cursor.execute(query_str, tables_args + qu2) + cursor.execute('SELECT * FROM (' + query_str + ') AS ' \ + '"' + self._table + '" ' \ + 'LIMIT %d' % cursor.IN_MAX, tables_args + qu2) + + datas = cursor.dictfetchall() + cache_ctx = context.copy() + for i in ('_timestamp', '_delete', '_create_records', + '_delete_records'): + if i in cache_ctx: + del cache_ctx[i] + cache = cursor.cache.setdefault(repr(cache_ctx), {}) + cache.setdefault(self._name, {}) + for data in datas: + for i in data.keys(): + if i in ('_timestamp', '_datetime'): + continue + field = self._columns[i] + if field._type in ('many2one',): + if field.model_name not in self.pool.object_name_list(): + del data[i] + continue + model = self.pool.get(field.model_name) + if not data[i] and not (isinstance(data[i], (int, long)) + and not isinstance(data[i], type(False))): + data[i] = BrowseRecordNull() + else: + ctx = context + if hasattr(field, 'datetime_field') \ + and field.datetime_field: + ctx = context.copy() + if field.datetime_field not in data: + del data[i] + continue + ctx['_datetime'] = data[field.datetime_field] + data[i] = BrowseRecord(cursor, user, data[i], + model, context=ctx) + cache[self._name].setdefault(data['id'], {}) + cache[self._name][data['id']].update(data) + + if len(datas) >= cursor.IN_MAX: + select_fields2 = [select_fields[0]] + if self._history and context.get('_datetime') \ + and not query_string: + select_fields2 += [select_fields[1]] + cursor.execute('SELECT * FROM (' \ + 'SELECT ' + ','.join(select_fields2) + ' FROM ' + \ + ' '.join(tables) + (qu1 and ' WHERE ' + qu1 or '') + \ + (order_by and ' ORDER BY ' + order_by or '') + \ + limit_str + offset_str + ') AS "' + self._table + '" ' \ + 'OFFSET %d' % cursor.IN_MAX, tables_args + qu2) + datas += cursor.dictfetchall() if self._history and context.get('_datetime'): res = [] ids_date = {} - for row in cursor.fetchall(): - if row[0] in ids_date: - if row[1] <= ids_date[row[0]]: + for data in datas: + if data['id'] in ids_date: + if data['_datetime'] <= ids_date[data['id']]: continue - if row[0] in res: - res.remove(row[0]) - res.append(row[0]) - ids_date[row[0]] = row[1] + if data['id'] in res: + res.remove(data['id']) + res.append(data['id']) + ids_date[data['id']] = data['_datetime'] return res - res = cursor.fetchall() - return [x[0] for x in res] + return [x['id'] for x in datas] def search_domain(self, cursor, user, domain, active_test=True, context=None): ''' diff --git a/trytond/model/modelstorage.py b/trytond/model/modelstorage.py index 382885ac77..adf1cdd4a5 100644 --- a/trytond/model/modelstorage.py +++ b/trytond/model/modelstorage.py @@ -398,12 +398,37 @@ class ModelStorage(Model): ''' ids = self.search(cursor, user, domain, offset=offset, limit=limit, order=order, context=context) + + if not fields_names: + fields_names = list(set(self._columns.keys() \ + + self._inherit_fields.keys())) + + res = [] + for model in self.browse(cursor, user, ids, context=context): + record = {} + for fields_name in set(fields_names): + fields = fields_name.split('.') + while fields: + field_name = fields.pop() + if fields_name not in record: + record[fields_name] = model[field_name] + else: + if isinstance(record[fields_name], BrowseRecordNull): + continue + record[fields_name] = record[fields_name][field_name] + if isinstance(record[fields_name], BrowseRecordNull): + record[fields_name] = False + elif isinstance(record[fields_name], BrowseRecord): + record[fields_name] = record[fields_name].id + elif isinstance(record[fields_name], BrowseRecordList): + record[fields_name] = [x.id for x in record[fields_name]] + res.append(record) + if limit == 1: if not ids: return [] - ids = ids[0] - return self.read(cursor, user, ids, fields_names=fields_names, - context=context) + return res[0] + return res def _search_domain_active(self, domain, active_test=True, context=None): if context is None: