From 1e38fe6eb2895d6f6da14819b5e26e51d020765d Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 28 Feb 2008 12:37:33 +0100 Subject: [PATCH 1/5] Copy vals dict before modifying it --- trytond/osv/orm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trytond/osv/orm.py b/trytond/osv/orm.py index 96227485d7..456be0d59b 100644 --- a/trytond/osv/orm.py +++ b/trytond/osv/orm.py @@ -1496,6 +1496,9 @@ class ORM(object): user = user id vals = dictionary of the form {'field_name': field_value, ...} """ + + vals = vals.copy() + self.pool.get('ir.model.access').check(cursor, user, self._name, 'create') From 280dbc936512c6ee8c71d8e6100fa5ab85ad9a5a Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 28 Feb 2008 12:39:00 +0100 Subject: [PATCH 2/5] Little bugfix: Unlink on user now handle list of ids --- trytond/res/user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trytond/res/user.py b/trytond/res/user.py index c6db69ae36..106bea38d0 100644 --- a/trytond/res/user.py +++ b/trytond/res/user.py @@ -113,6 +113,8 @@ class User(OSV): return res def unlink(self, cursor, user, ids, context=None): + if isinstance(ids, (int, long)): + ids = [ids] if 1 in ids: raise ExceptOSV('UserError', 'You can not remove the root user\n' \ From 878a762016ea4121264486aac38a27a338a7f7a8 Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 28 Feb 2008 13:36:28 +0100 Subject: [PATCH 3/5] Move all import stuff out of model.py and put it in convert.py. Other modifications: * less memory hungry implementation. * now delete record in the reverse order of creation. * Workflow transtion are deleted only if the corresponding activity is deleted. --- trytond/ir/model.py | 278 +------------------------- trytond/module.py | 29 ++- trytond/tools/convert.py | 407 ++++++++++++++++++++++++++++++++++----- 3 files changed, 390 insertions(+), 324 deletions(-) diff --git a/trytond/ir/model.py b/trytond/ir/model.py index cfd6cafb18..d004b0f427 100644 --- a/trytond/ir/model.py +++ b/trytond/ir/model.py @@ -1,20 +1,10 @@ "model" from trytond.osv import fields, OSV -from trytond.netsvc import Logger, LocalService, LOG_ERROR, LOG_INFO, LOG_WARNING, LOG_DEBUG from trytond.osv.orm import ExceptORM from trytond.tools import Cache import time -# Custom exception: - -class Unhandled_field(Exception): - """ - Raised when a field type is not supported by the update mechanism. - """ - pass - - class Model(OSV): "Model" _name = 'ir.model' @@ -186,271 +176,17 @@ class ModelData(OSV): self.fs2values = None - def populate_fs2db(self, cursor, user): - """Fetch all the db_id, model tuple in the table in one shot - (if necessary). This table is kept as a cache for the future - conversions between fs_id abd db_id""" - - if not self.fs2db: self.fs2db = {} - if cursor.dbname not in self.fs2db: - module_data_ids = self.search(cursor, user,[]) - self.fs2db[cursor.dbname] = dict([ - ((x.fs_id, x.module), - (x.db_id, x.model, x.id)) for x in \ - self.browse(cursor, user, module_data_ids)]) - - def populate_fs2values(self, cursor, user): - """Fetch all the table in one shot (if necessary). When a - record is imported ny import_record, the corresponding item in - fs2values is removed. When post_import is called, the items - remaining in fs2values are the one that were in the model_data - table but not in the fs, so they are removed.""" - # Improvement: store the fiedls names and a hash of the values - # instead of all the values. - if not self.fs2values: self.fs2values = {} - if cursor.dbname not in self.fs2values: - module_data_ids = self.search(cursor, user,[]) - self.fs2values[cursor.dbname] = dict([ - ((x.fs_id, x.module), - x.values) for x in \ - self.browse(cursor, user, module_data_ids)]) - def get_id(self, cursor, user, module, fs_id): - - # Pre-Fetch all the data at once - self.populate_fs2db(cursor, user) - - # return only the db_id - if (fs_id, module) in self.fs2db[cursor.dbname]: - return self.fs2db[cursor.dbname][(fs_id, module)][0] - else: - raise Exception("Reference to %s not found"%\ - ".".join([module,fs_id])) - - @staticmethod - def _clean_value(key, browse_record, object_ref): """ - Take a field name, a browse_record, and a reference to the - corresponding object. Return a raw value has it must look on the - db. + Return for an fs_id the corresponding db_id. """ + ids = self.search(cursor, user, + [('module','=',module),('fs_is','=',fs_id)]) + if not ids: + raise Exception("Reference to %s not found"% \ + ".".join([module,fs_id])) + return self.read(cursor, user, ids[0],['db_id'])['db_id'] - # search the field type in the object or in a parent - if key in object_ref._columns: - field_type = object_ref._columns[key]._type - else: - field_type = object_ref._inherit_fields[key][2]._type - - # handle the value regarding to the type - if field_type == 'many2one': - return browse_record[key] and browse_record[key].id or False - elif field_type in ['one2one', 'one2many', "many2many"]: - raise Unhandled_field() - else: - return browse_record[key] - - - def import_record(self, cursor, user, model, module, values, fs_id): - - if not fs_id: - raise Exception('import_record : Argument fs_id is mandatory') - - if '.' in fs_id: - assert len(fs_id.split('.')) == 2, '"%s" contains too many dots. '\ - 'file system ids should contain ot most one dot ! ' \ - 'These are used to refer to other modules data, ' \ - 'as in module.reference_id' % (fs_id) - - module, fs_id = fs_id.split('.') - - self.populate_fs2db(cursor, user) - self.populate_fs2values(cursor, user) - - object_ref = self.pool.get(model) - - if (fs_id, module) in self.fs2db[cursor.dbname]: - # this record is already in the db: - db_id, db_model, mdata_id = \ - self.fs2db[cursor.dbname][(fs_id, module)] - old_values = self.fs2values[cursor.dbname][(fs_id, module)] - if old_values == None: - old_values = {} - else: - old_values = eval(old_values) - - - # Check if values for this record has been modified in the - # db, if not it's ok to overwrite them. - if model != db_model: - raise Exception("This record try to overwrite" - "data with the wrong model: %s (module: %s)"% (fs_id, module)) - - #Re-create object if it was deleted - if not object_ref.search(cursor, user, [('id', '=', db_id)]): - db_id = object_ref.create(cursor, user, values, - context={'module': module}) - data_id = self.search(cursor, user, [ - ('fs_id', '=', fs_id), - ('module', '=', module), - ], limit=1)[0] - self.write(cursor, user, data_id, { - 'db_id': db_id, - }) - self.fs2db[cursor.dbname][(fs_id, module)] = \ - (db_id, db_model, mdata_id) - - db_values = object_ref.browse(cursor, user, db_id) - - to_update = {} - for key in values: - - try: - db_field = ModelData._clean_value(key, db_values, - object_ref) - except Unhandled_field: - logger = Logger() - logger.notify_channel('init', LOG_WARNING, - 'Field %s on %s : integrity not tested.'%(key, model)) - to_update[key] = values[key] - continue - - # if the fs value is the same has in the db, whe ignore it - if db_field == values[key]: - continue - - # we cannot update a field if it was changed by a user - if key not in old_values: - default_value = object_ref._defaults.get( - key, lambda *a:None)(cursor, user) - if db_field != default_value: - logger = Logger() - logger.notify_channel('init', LOG_WARNING, - "Field %s of %s@%s not updated (id: %s), because "\ - "it has changed since the last update"% \ - (key, db_id, model, fs_id)) - continue - - elif (old_values[key] and db_field) and \ - old_values[key] != db_field: - logger = Logger() - logger.notify_channel('init', LOG_WARNING, - "Field %s of %s@%s not updated (id: %s), because "\ - "it has changed since the last update."%\ - (key, db_id, model, fs_id)) - continue - - # so, the field in the fs and in the db are different, - # and no user changed the value in the db: - to_update[key] = values[key] - - # if there is values to update: - if to_update: - # write the values in the db: - object_ref.write(cursor, user, db_id, to_update) - # re-read it: this ensure that we store the real value - # in the model_data table: - db_val = object_ref.browse(cursor, user, db_id) - for key in to_update: - try: - values[key] = ModelData._clean_value( - key, db_val, object_ref) - except Unhandled_field: - continue - - if values != old_values: - self.write(cursor, user, mdata_id, { - 'fs_id': fs_id, - 'model': model, - 'module': module, - 'db_id': db_id, - 'values': str(values), - 'date_update': time.strftime('%Y-%m-%d %H:%M:%S'), - }) - - # Remove this record from the value list. This means that - # the corresponding record have been found. - self.fs2values[cursor.dbname][(fs_id, module)] = None - else: - # this record is new, create it in the db: - db_id = object_ref.create(cursor, user, values, - context={'module': module}) - # re-read it: this ensure that we store the real value - # in the model_data table: - db_val = object_ref.browse(cursor, user, db_id) - for key in values: - try: - values[key] = ModelData._clean_value(key, db_val, - object_ref) - except Unhandled_field: - continue - - mdata_id = self.create(cursor, user, { - 'fs_id': fs_id, - 'model': model, - 'module': module, - 'db_id': db_id, - 'values': str(values), - }) - # update fs2db: - self.fs2db[cursor.dbname][(fs_id, module)]= (db_id, model, - mdata_id) - self.fs2values[cursor.dbname][(fs_id, module)] = None - - def post_import(self, cursor, user, modules): - # Test because of a wrong extra call. See todo at the end of - # load_module_graph in module.py - # Globaly this function is a bit dirty. - if not (self.fs2values and self.fs2values.get(cursor.dbname)): - return True - - wf_service = LocalService("workflow") - - mdata_unlink = [] - - for (fs_id, module) in self.fs2values[cursor.dbname]: - if self.fs2values[cursor.dbname][(fs_id, module)] == None: - continue - if module in modules: - (db_id, model, mdata_id) = self.fs2db[cursor.dbname]\ - [(fs_id, module)] - - if model == 'workflow.activity': - cursor.execute('SELECT res_type, db_id ' \ - 'FROM wkf_instance ' \ - 'WHERE id IN (' \ - 'SELECT instance FROM wkf_workitem ' \ - 'WHERE act_id = %d)', (db_id,)) - wkf_todo.extend(cursor.fetchall()) - cursor.execute("UPDATE wkf_transition " \ - "SET condition = 'True', role_id = NULL, " \ - "signal = NULL, act_to = act_from, " \ - "act_from = %d " \ - "WHERE act_to = %d", (db_id, db_id)) - cursor.execute("DELETE FROM wkf_transition " \ - "WHERE act_to = %d", (db_id,)) - - wf_service.trg_write(user, model, db_id, cursor) - - logger = Logger() - logger.notify_channel('init', LOG_INFO, - 'Deleting %s@%s' % (db_id, model)) - try: - self.pool.get(model).unlink(cursor, user, db_id) - mdata_unlink.append(mdata_id) - except: - raise - logger.notify_channel('init', LOG_ERROR, - 'Could not delete id: %d of model %s\n' \ - 'There should be some relation ' \ - 'that points to this resource\n' \ - 'You should manually fix this ' \ - 'and restart --update=module' % \ - (db_id, model)) - - if mdata_unlink: - self.unlink(cursor, user, mdata_unlink) - del self.fs2values[cursor.dbname] - return True ModelData() diff --git a/trytond/module.py b/trytond/module.py index 9494da542e..297982af65 100644 --- a/trytond/module.py +++ b/trytond/module.py @@ -176,7 +176,7 @@ def init_module_wizards(cursor, module_name, wizard_list): def load_module_graph(cursor, graph, lang=None): if lang is None: lang = ['en_US'] - package_todo = [] + modules_todo = [] for package in graph: module = package.name Logger().notify_channel('init', LOG_INFO, 'module:%s' % module) @@ -199,6 +199,13 @@ def load_module_graph(cursor, graph, lang=None): init_module_wizards(cursor, module, wizards) demo = hasattr(package, 'demo') \ or (package_demo and package_state != 'installed') + + #Instanciate a new parser for the package: + tryton_parser = tools.TrytondXmlHandler( + cursor=cursor, + pool=pooler.get_pool(cursor.dbname), + module=module,) + for filename in package.datas.get('xml', []): mode = 'update' if hasattr(package, 'init') or package_state=='to install': @@ -215,8 +222,12 @@ def load_module_graph(cursor, graph, lang=None): if new_query: cursor.execute(new_query) else: - tools.convert_xml_import(cursor, module, - tools.file_open(OPJ(module, filename))) + # Feed the parser with xml content: + tryton_parser.parse_xmlstream( + tools.file_open(OPJ(module, filename))) + + modules_todo.append((module, tryton_parser.to_delete)) + for filename in package.datas.get('translation', []): lang2 = os.path.splitext(filename)[0] if lang2 not in lang: @@ -229,14 +240,18 @@ def load_module_graph(cursor, graph, lang=None): if demo: cursor.execute('UPDATE ir_module_module SET demo = %s ' \ 'WHERE name = %s', (True, package.name)) - package_todo.append(package.name) + cursor.execute("UPDATE ir_module_module SET state = 'installed' " \ "WHERE name = %s", (package.name,)) cursor.commit() - pool = pooler.get_pool(cursor.dbname) - # TODO : post_import is called even if there not init nor update - pool.get('ir.model.data').post_import(cursor, 0, package_todo) + + # Vacuum : + while modules_todo: + (module, to_delete) = modules_todo.pop() + tools.post_import(cursor, module, to_delete) + + cursor.commit() def get_module_list(): diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index 95c60fb1fb..16fc8bba82 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -5,21 +5,16 @@ from trytond import pooler from trytond.osv.fields import Integer import csv import os.path -from trytond.netsvc import Logger, LOG_ERROR, LOG_INFO, LocalService +from trytond.netsvc import Logger, LOG_ERROR, LOG_INFO, LocalService, LOG_WARNING from trytond.config import CONFIG from trytond.version import VERSION import logging +import time +from xml import sax CDATA_START = re.compile('^\s*\<\!\[cdata\[', re.IGNORECASE) CDATA_END = re.compile('\]\]\>\s*$', re.IGNORECASE) -# TODO : -# Add some exception helper, and give a good diagnostic for the errors - - -from xml import sax - - class DummyTagHandler: """Dubhandler implementing empty methods. Will be used when whe want to ignore the xml content""" @@ -133,9 +128,8 @@ class MenuitemTagHandler: return self else: - res = self.mh.pool.get('ir.model.data').import_record( - self.mh.cursor, self.mh.user, - 'ir.ui.menu', self.mh.module, self.values, self.xml_id) + res = self.mh.import_record( + 'ir.ui.menu', self.values, self.xml_id) return None def current_state(self): @@ -264,18 +258,72 @@ class RecordTagHandler: return self elif name == "record": - res = self.mh.pool.get('ir.model.data').import_record( - self.mh.cursor, self.mh.user, - self.model._name, self.mh.module, self.values, self.xml_id) + res = self.mh.import_record( + self.model._name, self.values, self.xml_id) return None else: raise Exception("Unexpected closing tag '%s'"% (name,)) def current_state(self): - return "Tag record for model %s with id: %s"% \ + return "Last tag record seen: model %s with id %s."% \ (self.model and self.model._name or "?", self.xml_id) + +# Custom exception: +class Unhandled_field(Exception): + """ + Raised when a field type is not supported by the update mechanism. + """ + pass + +class Fs2bdAccessor: + """ + Used in TrytondXmlHandler. + Provide some helper function to ease cache access and management. + """ + + def __init__(self, cursor, user, modeldata_obj): + self.fs2db = {} + self.fetched_modules = [] + self.modeldata_obj = modeldata_obj + self.cursor = cursor + self.user = user + + def get(self, module, fs_id): + if module not in self.fetched_modules: + self.fetch_new_module(module) + return self.fs2db[module].get(fs_id, None) + + def set(self, module, fs_id, values): + """ + Whe call the prefetch function here to. Like that whe are sure + not to erase data when get is called. + """ + if not module in self.fetched_modules: + self.fetch_new_module(module) + if fs_id not in self.fs2db[module]: + self.fs2db[module][fs_id] = {} + fs2db_val = self.fs2db[module][fs_id] + for key, val in values.items(): + fs2db_val[key] = val + + def fetch_new_module(self, module): + if module == "ir.ui.menu": raise + self.fs2db[module] = {} + module_data_ids = self.modeldata_obj.search( + self.cursor, self.user, [('module','=',module)] + ) + for rec in self.modeldata_obj.browse( + self.cursor, self.user, module_data_ids): + + self.fs2db[rec.module][rec.fs_id] = { + "db_id": rec.db_id, "model": rec.model, + "id": rec.id, "values": rec.values + } + self.fetched_modules.append(module) + + class TrytondXmlHandler(sax.handler.ContentHandler): def __init__(self, cursor, pool, module,): @@ -285,7 +333,9 @@ class TrytondXmlHandler(sax.handler.ContentHandler): self.cursor = cursor self.user = 0 self.module = module - + self.modeldata_obj = pool.get('ir.model.data') + self.fs2db = Fs2bdAccessor(cursor, self.user, self.modeldata_obj) + self.to_delete = self.populate_to_delete() # Tag handlders are used to delegate the processing self.taghandlerlist = { @@ -297,15 +347,32 @@ class TrytondXmlHandler(sax.handler.ContentHandler): # Managed tags are handled by the current class self.managedtags= ["data", "tryton"] + # Connect to the sax api: + self.sax_parser = sax.make_parser() + # Tell the parser we are not interested in XML namespaces + self.sax_parser.setFeature(sax.handler.feature_namespaces, 0) + self.sax_parser.setContentHandler(self) - def get_id(self, xml_id): - module = self.module - if '.' in xml_id: - module, xml_id = xml_id.split('.') + def parse_xmlstream(self, stream): + """ + Take a byte stream has input and parse the xml content. + """ - return self.pool.get('ir.model.data').get_id( - self.cursor, self.user, module, xml_id) + source = sax.InputSource() + source.setByteStream(stream) + + try: + self.sax_parser.parse(source) + except: + Logger().notify_channel( + "init", LOG_ERROR, + "Error while parsing xml file. Current state is:\n" +\ + self.current_state() + ) + + raise + return self.to_delete def startElement(self, name, attributes): """Rebind the current handler if necessary and call @@ -347,35 +414,283 @@ class TrytondXmlHandler(sax.handler.ContentHandler): else: return "In top level tag" -def convert_xml_import_sax(cursor, module, xmlstream): + def get_id(self, xml_id): - parser = sax.make_parser() - # Tell the parser we are not interested in XML namespaces - parser.setFeature(sax.handler.feature_namespaces, 0) + if '.' in xml_id: + module, xml_id = xml_id.split('.') + else: + module = self.module - handler = TrytondXmlHandler( - cursor=cursor, - pool=pooler.get_pool(cursor.dbname), - module=module, - ) + if self.fs2db.get(module, xml_id) == None: + raise Exception("Reference to %s not found"% \ + ".".join([module,xml_id])) + return self.fs2db.get(module, xml_id)["db_id"] - parser.setContentHandler(handler) - source = sax.InputSource() - source.setByteStream(xmlstream) - try: - parser.parse(source) - except: - Logger().notify_channel( - "init", LOG_ERROR, - "Error while parsing xml file. Current state is:\n" +\ - handler.current_state() + + @staticmethod + def _clean_value(key, browse_record, object_ref): + """ + Take a field name, a browse_record, and a reference to the + corresponding object. Return a raw value has it must look on the + db. + """ + + # search the field type in the object or in a parent + if key in object_ref._columns: + field_type = object_ref._columns[key]._type + else: + field_type = object_ref._inherit_fields[key][2]._type + + # handle the value regarding to the type + if field_type == 'many2one': + return browse_record[key] and browse_record[key].id or False + elif field_type in ['one2one', 'one2many', "many2many"]: + raise Unhandled_field() + else: + return browse_record[key] + + + def populate_to_delete(self): + """Create a list of all the records that whe should met in the update + process. The records that are not encountered are deleted from the + database in post_import.""" + + module_data_ids = self.modeldata_obj.search( + self.cursor, self.user, [('module','=',self.module)] ) + return [(rec.id,rec.model,rec.db_id) for rec in self.modeldata_obj.browse( + self.cursor, self.user, module_data_ids)] - raise + def import_record(self, model, values, fs_id): + + cursor = self.cursor + module = self.module + user = self.user + + if not fs_id: + raise Exception('import_record : Argument fs_id is mandatory') + + if '.' in fs_id: + assert len(fs_id.split('.')) == 2, '"%s" contains too many dots. '\ + 'file system ids should contain ot most one dot ! ' \ + 'These are used to refer to other modules data, ' \ + 'as in module.reference_id' % (fs_id) + + module, fs_id = fs_id.split('.') + + object_ref = self.pool.get(model) + + if self.fs2db.get(module, fs_id): + # this record is already in the db: + # XXX maybe use only one call to get() + db_id, db_model, mdata_id, old_values = \ + [self.fs2db.get(module, fs_id)[x] for x in \ + ["db_id","model","id","values"] + ] + + if old_values == None: + old_values = {} + else: + old_values = eval(old_values) + + + # Check if values for this record has been modified in the + # db, if not it's ok to overwrite them. + if model != db_model: + raise Exception("This record try to overwrite" + "data with the wrong model: %s (module: %s)"% (fs_id, module)) + + #Re-create object if it was deleted + if not object_ref.search(cursor, user, [('id', '=', db_id)]): + db_id = object_ref.create(cursor, user, values, + context={'module': module}) + data_id = self.modeldata_obj.search(cursor, user, [ + ('fs_id', '=', fs_id), + ('module', '=', module), + ], limit=1)[0] + self.modeldata_obj.write(cursor, user, data_id, { + 'db_id': db_id, + }) + self.fs2db.get(module, fs_id)["db_id"] = db_id + + db_val = object_ref.browse(cursor, user, db_id) + + to_update = {} + for key in values: + + try: + db_field = self._clean_value(key, db_val, object_ref) + except Unhandled_field: + logger = Logger() + logger.notify_channel('init', LOG_WARNING, + 'Field %s on %s : integrity not tested.'%(key, model)) + to_update[key] = values[key] + continue + + # if the fs value is the same has in the db, whe ignore it + if db_field == values[key]: + continue + + # we cannot update a field if it was changed by a user + if key not in old_values: + default_value = object_ref._defaults.get( + key, lambda *a:None)(cursor, user) + if db_field != default_value: + logger = Logger() + + logger.notify_channel('init', LOG_WARNING, + "Field %s of %s@%s not updated (id: %s), because "\ + "it has changed since the last update"% \ + (key, db_id, model, fs_id)) + continue + + elif (old_values[key] and db_field) and \ + old_values[key] != db_field: + logger = Logger() + logger.notify_channel('init', LOG_WARNING, + "Field %s of %s@%s not updated (id: %s), because "\ + "it has changed since the last update."%\ + (key, db_id, model, fs_id)) + continue + + # so, the field in the fs and in the db are different, + # and no user changed the value in the db: + to_update[key] = values[key] + + # if there is values to update: + if to_update: + # write the values in the db: + object_ref.write(cursor, user, db_id, to_update) + # re-read it: this ensure that we store the real value + # in the model_data table: + db_val = object_ref.browse(cursor, user, db_id) + for key in to_update: + try: + values[key] = self._clean_value( + key, db_val, object_ref) + except Unhandled_field: + continue + + if values != old_values: + self.modeldata_obj.write(cursor, user, mdata_id, { + 'fs_id': fs_id, + 'model': model, + 'module': module, + 'db_id': db_id, + 'values': str(values), + 'date_update': time.strftime('%Y-%m-%d %H:%M:%S'), + }) + + # Remove this record from the to_delete list. This means that + # the corresponding record have been found. + if module == self.module: + self.to_delete.remove((mdata_id, model, db_id)) + else: + # this record is new, create it in the db: + db_id = object_ref.create(cursor, user, values, + context={'module': module}) + # re-read it: this ensure that we store the real value + # in the model_data table: + db_val = object_ref.browse(cursor, user, db_id) + for key in values: + try: + values[key] = self._clean_value(key, db_val, + object_ref) + except Unhandled_field: + continue + + mdata_id = self.modeldata_obj.create(cursor, user, { + 'fs_id': fs_id, + 'model': model, + 'module': module, + 'db_id': db_id, + 'values': str(values), + }) + # update fs2db: + self.fs2db.set(module, fs_id, { + "db_id": db_id, "model": model, + "id": mdata_id, "values": str(values)}) + + +def post_import(cursor, module, to_delete): + """ + Remove the records that are given in to_delete. + """ + + user = 0 + wf_service = LocalService("workflow") + mdata_unlink = [] + pool = pooler.get_pool(cursor.dbname) + modeldata_obj = pool.get("ir.model.data") + + # Sort by model_data id descending and thus avoiding most + # dependency problems: + to_delete.sort(key= lambda x : -x[0]) + + for mdata_id, model,db_id in to_delete: + + # Whe skip transitions, they will be deleted with the + # corresponding activity: + if model == 'workflow.transition': + continue + + if model == 'workflow.activity': + + wkf_todo = [] + # search for records that are in the state/activity that + # we want to delete... + cursor.execute('SELECT res_type, db_id ' \ + 'FROM wkf_instance ' \ + 'WHERE id IN (' \ + 'SELECT instance FROM wkf_workitem ' \ + 'WHERE act_id = %d)', (db_id,)) + #... connect the transitions backward... + wkf_todo.extend(cursor.fetchall()) + cursor.execute("UPDATE wkf_transition " \ + "SET condition = 'True', role_id = NULL, " \ + "signal = NULL, act_to = act_from, " \ + "act_from = %d " \ + "WHERE act_to = %d", (db_id, db_id)) + # ... and force the record to follow them: + for wkf_model,wkf_model_id in wkf_todo: + wf_service = netsvc.LocalService("workflow") + wf_service.trg_write(uid, model, id, cr) + + # Collect the ids of these transition in model_data + cursor.execute( + "SELECT md.id FROM ir_model_data md " \ + "JOIN wkf_transition t ON "\ + "(md.model='workflow.transition' and md.db_id=t.id)" \ + "WHERE t.act_to = %d", (db_id,)) + mdata_unlink.extend([x[0] for x in cr.fetchall()]) + + # And finally delete the transitions + cursor.execute("DELETE FROM wkf_transition " \ + "WHERE act_to = %d", (db_id,)) + + wf_service.trg_write(user, model, db_id, cursor) + + + logger = Logger() + logger.notify_channel('init', LOG_INFO, + 'Deleting %s@%s' % (db_id, model)) + try: + # Deletion of the record + pool.get(model).unlink(cursor, user, db_id) + mdata_unlink.append(mdata_id) + except: + raise + logger.notify_channel('init', LOG_ERROR, + 'Could not delete id: %d of model %s\n' \ + 'There should be some relation ' \ + 'that points to this resource\n' \ + 'You should manually fix this ' \ + 'and restart --update=module' % \ + (db_id, model)) + + # Clean model_data: + if mdata_unlink: + modeldata_obj.unlink(cursor, user, mdata_unlink) return True - - -# use convert_xml_import_sax or convert_xml_import_dom -convert_xml_import = convert_xml_import_sax From f4a69ffb2520afdde196b660ba15ed2d0da10ef5 Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 28 Feb 2008 14:14:24 +0100 Subject: [PATCH 4/5] Order the to_delete list at read time instead of sorting it later --- trytond/tools/convert.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index 354e63fe18..d4fea4678c 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -456,8 +456,11 @@ class TrytondXmlHandler(sax.handler.ContentHandler): process. The records that are not encountered are deleted from the database in post_import.""" + # Fetch the data in id descending order to avoid depedendcy + # problem when the corresponding recordds will be deleted: module_data_ids = self.modeldata_obj.search( - self.cursor, self.user, [('module','=',self.module)] + self.cursor, self.user, [('module','=',self.module)], + order="id desc", ) return [(rec.id,rec.model,rec.db_id) for rec in self.modeldata_obj.browse( self.cursor, self.user, module_data_ids)] @@ -624,10 +627,6 @@ def post_import(cursor, module, to_delete): pool = pooler.get_pool(cursor.dbname) modeldata_obj = pool.get("ir.model.data") - # Sort by model_data id descending and thus avoiding most - # dependency problems: - to_delete.sort(key= lambda x : -x[0]) - for mdata_id, model,db_id in to_delete: # Whe skip transitions, they will be deleted with the From 0dd3501c62cff7fa6c4838cf95a36716bdee3c59 Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 28 Feb 2008 16:57:38 +0100 Subject: [PATCH 5/5] Use xml_id to store the records to delete instead of the former tuple. This provide more robust updates. --- trytond/tools/convert.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index d4fea4678c..5563c23102 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -462,7 +462,7 @@ class TrytondXmlHandler(sax.handler.ContentHandler): self.cursor, self.user, [('module','=',self.module)], order="id desc", ) - return [(rec.id,rec.model,rec.db_id) for rec in self.modeldata_obj.browse( + return [rec.fs_id for rec in self.modeldata_obj.browse( self.cursor, self.user, module_data_ids)] def import_record(self, model, values, fs_id): @@ -588,7 +588,7 @@ class TrytondXmlHandler(sax.handler.ContentHandler): # Remove this record from the to_delete list. This means that # the corresponding record have been found. if module == self.module: - self.to_delete.remove((mdata_id, model, db_id)) + self.to_delete.remove(fs_id) else: # this record is new, create it in the db: db_id = object_ref.create(cursor, user, values, @@ -627,7 +627,13 @@ def post_import(cursor, module, to_delete): pool = pooler.get_pool(cursor.dbname) modeldata_obj = pool.get("ir.model.data") - for mdata_id, model,db_id in to_delete: + mdata_ids = modeldata_obj.search( + cursor, user, [('fs_id','in',to_delete)], + order="id desc", + ) + + for mrec in modeldata_obj.browse(cursor, user, mdata_ids): + mdata_id, model,db_id = mrec.id, mrec.model, mrec.db_id # Whe skip transitions, they will be deleted with the # corresponding activity: