From 8c1337ab88ff76a8efecf111e84bdc8e58a0ba1f Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Sun, 30 Dec 2007 00:18:11 +0100 Subject: [PATCH 1/9] WIP sax api: added support for eval attributes, use logger instead of print for error message --- trytond/ir/actions.xml | 32 ++++++++++++------------- trytond/ir/attachment.xml | 8 +++---- trytond/ir/cron.xml | 8 +++---- trytond/ir/exports.xml | 8 +++---- trytond/ir/model.xml | 24 +++++++++---------- trytond/ir/module/category.xml | 8 +++---- trytond/ir/module/module.xml | 8 +++---- trytond/ir/module/repository.xml | 8 +++---- trytond/ir/rule.xml | 16 ++++++------- trytond/ir/sequence.xml | 18 +++++++------- trytond/ir/translation.xml | 8 +++---- trytond/ir/values.xml | 8 +++---- trytond/res/lang.xml | 4 ++-- trytond/res/request.xml | 24 +++++++++---------- trytond/res/users.xml | 30 ++++++++++++------------ trytond/tools/convert.py | 28 +++++++++++++++------- trytond/workflow/workflow.xml | 40 ++++++++++++++++---------------- 17 files changed, 146 insertions(+), 134 deletions(-) diff --git a/trytond/ir/actions.xml b/trytond/ir/actions.xml index f1a4997759..866f3e44fc 100644 --- a/trytond/ir/actions.xml +++ b/trytond/ir/actions.xml @@ -7,7 +7,7 @@ ir.actions.actions form - @@ -22,7 +22,7 @@ - ]]> + ]]> @@ -30,12 +30,12 @@ ir.actions.actions tree - - ]]> + ]]> @@ -65,7 +65,7 @@ ir.actions.report form - @@ -73,7 +73,7 @@ - ]]> + ]]> @@ -81,13 +81,13 @@ ir.actions.report tree - - ]]> + ]]> @@ -117,7 +117,7 @@ ir.actions.act_window form - @@ -145,7 +145,7 @@ - ]]> + ]]> @@ -153,7 +153,7 @@ ir.actions.act_window tree - @@ -161,7 +161,7 @@ - ]]> + ]]> @@ -189,14 +189,14 @@ ir.actions.wizard form - - ]]> + ]]> @@ -204,14 +204,14 @@ ir.actions.wizard tree - - ]]> + ]]> diff --git a/trytond/ir/attachment.xml b/trytond/ir/attachment.xml index 61c5be3ff3..c0987edc89 100644 --- a/trytond/ir/attachment.xml +++ b/trytond/ir/attachment.xml @@ -6,7 +6,7 @@ ir.attachment form - @@ -17,7 +17,7 @@ - ]]> + ]]> @@ -25,13 +25,13 @@ ir.attachment tree - - ]]> + ]]> diff --git a/trytond/ir/cron.xml b/trytond/ir/cron.xml index 3dd72333b1..61819a90eb 100644 --- a/trytond/ir/cron.xml +++ b/trytond/ir/cron.xml @@ -6,7 +6,7 @@ ir.cron tree - @@ -14,7 +14,7 @@ - ]]> + ]]> @@ -22,7 +22,7 @@ ir.cron form - @@ -41,7 +41,7 @@ - ]]> + ]]> diff --git a/trytond/ir/exports.xml b/trytond/ir/exports.xml index 4e8b10c4cc..4ad90c20ff 100644 --- a/trytond/ir/exports.xml +++ b/trytond/ir/exports.xml @@ -6,13 +6,13 @@ ir.exports form - - ]]> + ]]> @@ -20,12 +20,12 @@ ir.exports tree - - ]]> + ]]> diff --git a/trytond/ir/model.xml b/trytond/ir/model.xml index 5113b193cb..59941a332e 100644 --- a/trytond/ir/model.xml +++ b/trytond/ir/model.xml @@ -6,7 +6,7 @@ ir.model form - @@ -22,7 +22,7 @@ - ]]> + ]]> @@ -30,12 +30,12 @@ ir.model tree - - ]]> + ]]> @@ -64,7 +64,7 @@ ir.model.fields form - @@ -72,7 +72,7 @@ - ]]> + ]]> @@ -80,12 +80,12 @@ ir.model.fields tree - - ]]> + ]]> @@ -114,13 +114,13 @@ ir.model.access tree - - ]]> + ]]> @@ -128,7 +128,7 @@ ir.model.access form - @@ -139,7 +139,7 @@ - ]]> + ]]> diff --git a/trytond/ir/module/category.xml b/trytond/ir/module/category.xml index ef2b9321bf..1700819a0c 100644 --- a/trytond/ir/module/category.xml +++ b/trytond/ir/module/category.xml @@ -6,12 +6,12 @@ ir.module.category form - - ]]> + ]]> @@ -20,11 +20,11 @@ tree child_ids - - ]]> + ]]> diff --git a/trytond/ir/module/module.xml b/trytond/ir/module/module.xml index 28c4dc64bf..a019c8e528 100644 --- a/trytond/ir/module/module.xml +++ b/trytond/ir/module/module.xml @@ -7,7 +7,7 @@ ir.module.module form - @@ -51,7 +51,7 @@ - ]]> + ]]> @@ -59,7 +59,7 @@ ir.module.module tree - @@ -69,7 +69,7 @@ - ]]> + ]]> diff --git a/trytond/ir/module/repository.xml b/trytond/ir/module/repository.xml index a289c8b05b..88339687f0 100644 --- a/trytond/ir/module/repository.xml +++ b/trytond/ir/module/repository.xml @@ -6,7 +6,7 @@ ir.module.repository form - @@ -14,7 +14,7 @@ - ]]> + ]]> @@ -22,14 +22,14 @@ ir.module.repository tree - - ]]> + ]]> diff --git a/trytond/ir/rule.xml b/trytond/ir/rule.xml index d3d224c6af..fc1980d6a8 100644 --- a/trytond/ir/rule.xml +++ b/trytond/ir/rule.xml @@ -6,7 +6,7 @@ ir.rule.group form - @@ -19,7 +19,7 @@ colspan="6" align="0.0"/> - ]]> + ]]> @@ -27,13 +27,13 @@ ir.rule.group tree - - ]]> + ]]> @@ -62,7 +62,7 @@ ir.rule form - @@ -70,7 +70,7 @@ - ]]> + ]]> @@ -78,13 +78,13 @@ ir.rule tree - - ]]> + ]]> diff --git a/trytond/ir/sequence.xml b/trytond/ir/sequence.xml index 8ce92a0d04..3927633977 100644 --- a/trytond/ir/sequence.xml +++ b/trytond/ir/sequence.xml @@ -7,7 +7,7 @@ ir.sequence form - @@ -23,8 +23,8 @@ + ]]> + @@ -32,13 +32,13 @@ ir.sequence tree - - ]]> + ]]> @@ -69,12 +69,12 @@ ir.sequence.type form - - ]]> + ]]> @@ -82,12 +82,12 @@ ir.sequence.type tree - - ]]> + ]]> ir.translation form - @@ -15,7 +15,7 @@ - ]]> + ]]> @@ -23,7 +23,7 @@ ir.translation tree - @@ -31,7 +31,7 @@ - ]]> + ]]> diff --git a/trytond/ir/values.xml b/trytond/ir/values.xml index de99bd3c13..7db70eb195 100644 --- a/trytond/ir/values.xml +++ b/trytond/ir/values.xml @@ -6,7 +6,7 @@ ir.values form - @@ -18,7 +18,7 @@ - ]]> + ]]> @@ -26,7 +26,7 @@ ir.values tree - @@ -34,7 +34,7 @@ - ]]> + ]]> diff --git a/trytond/res/lang.xml b/trytond/res/lang.xml index 0a4bda68a6..b2b5a67dcf 100644 --- a/trytond/res/lang.xml +++ b/trytond/res/lang.xml @@ -10,7 +10,7 @@ res.lang tree - @@ -18,7 +18,7 @@ - ]]> + ]]> diff --git a/trytond/res/request.xml b/trytond/res/request.xml index 9c72d63d2e..0fb25201ea 100644 --- a/trytond/res/request.xml +++ b/trytond/res/request.xml @@ -6,7 +6,7 @@ res.request tree - @@ -16,7 +16,7 @@ - ]]> + ]]> @@ -24,7 +24,7 @@ res.request form - @@ -60,7 +60,7 @@ - ]]> + ]]> @@ -90,13 +90,13 @@ res.request.link form - - ]]> + ]]> @@ -104,12 +104,12 @@ res.request.link tree - - ]]> + ]]> @@ -117,14 +117,14 @@ res.request.history tree - - ]]> + ]]> @@ -132,7 +132,7 @@ res.request.history form - @@ -140,7 +140,7 @@ - ]]> + ]]> diff --git a/trytond/res/users.xml b/trytond/res/users.xml index 42bf3c8f18..96de8437f5 100644 --- a/trytond/res/users.xml +++ b/trytond/res/users.xml @@ -9,16 +9,16 @@ admin Administrator Administrator - - - + + + res.user.form res.user form - @@ -41,7 +41,7 @@ - ]]> + ]]> @@ -49,12 +49,12 @@ res.user tree - - ]]> + ]]> @@ -84,7 +84,7 @@ res.group form - @@ -98,7 +98,7 @@ - ]]> + ]]> @@ -106,11 +106,11 @@ res.group tree - - ]]> + ]]> @@ -140,12 +140,12 @@ res.role form - - ]]> + ]]> @@ -154,11 +154,11 @@ tree parent_id - - ]]> + ]]> diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index 36c3f71ea6..739c22b76a 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -10,8 +10,6 @@ from trytond.config import CONFIG from trytond.version import VERSION import logging - - CDATA_START = re.compile('^\s*\<\!\[cdata\[', re.IGNORECASE) CDATA_END = re.compile('\]\]\>\s*$', re.IGNORECASE) @@ -882,9 +880,24 @@ class RecordTagHandler: self.values[field_name] = self.mh.get_id(ref_attr) elif eval_attr: - print "NOT IMPLEMENTED : eval" - pass + import time + context = {} + context['time'] = time + context['version'] = VERSION.rsplit('.', 1)[0] + context['ref'] = self.mh.get_id + context['obj'] = lambda *a: 1 + try: + import pytz + except: + Logger().notify_channel("init", LOG_INFO, + 'could not find pytz library') + class Pytz(object): + all_timezones = [] + + pytz = Pytz() + context['pytz'] = pytz + self.values[field_name] = eval(eval_attr, context) else: raise Exception("Tags '%s' not supported inside tag record."% (name,)) @@ -995,9 +1008,8 @@ class TrytondXmlHandler(sax.handler.ContentHandler): pass else: - # TODO logger, logger, logger, ... - print "Tag", name , "not supported" - raise + Logger().notify_channel("init", LOG_INFO, + "Tag", name , "not supported") return else: self.taghandler.startElement(name, attributes) @@ -1043,4 +1055,4 @@ def convert_xml_import_sax(cursor, module, xmlstream, idref=None, mode='init', # use convert_xml_import_sax or convert_xml_import_dom -convert_xml_import = convert_xml_import_dom +convert_xml_import = convert_xml_import_sax diff --git a/trytond/workflow/workflow.xml b/trytond/workflow/workflow.xml index df5323bffa..07178f7c94 100644 --- a/trytond/workflow/workflow.xml +++ b/trytond/workflow/workflow.xml @@ -7,7 +7,7 @@ workflow form - @@ -15,7 +15,7 @@ - ]]> + ]]> @@ -23,13 +23,13 @@ workflow tree - - ]]> + ]]> @@ -58,7 +58,7 @@ workflow.activity form - @@ -94,7 +94,7 @@ - ]]> + ]]> @@ -102,7 +102,7 @@ workflow.activity tree - @@ -111,7 +111,7 @@ - ]]> + ]]> @@ -140,7 +140,7 @@ workflow.transition form - @@ -150,7 +150,7 @@ - ]]> + ]]> @@ -158,7 +158,7 @@ workflow.transition tree - @@ -168,7 +168,7 @@ - ]]> + ]]> @@ -197,7 +197,7 @@ workflow.instance form - @@ -205,7 +205,7 @@ - ]]> + ]]> @@ -213,7 +213,7 @@ workflow.instance tree - @@ -221,7 +221,7 @@ - ]]> + ]]> @@ -243,14 +243,14 @@ workflow.workitem form - - ]]> + ]]> @@ -258,14 +258,14 @@ workflow.workitem tree - - ]]> + ]]> From d87ad7141f47422080414658bc98458168fcdd7a Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 3 Jan 2008 00:32:25 +0100 Subject: [PATCH 2/9] WIP, Sax api : added handling of menuitems tags --- trytond/tools/convert.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index 739c22b76a..d4d95f4b1d 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -219,6 +219,7 @@ class XMLImport(object): if len(obj_id) > 64: Logger().notify_channel('init', LOG_ERROR, 'id: %s is to long (max: 64)' % xml_id) + def _tag_delete(self, cursor, rec, data_node=None): d_model = rec.getAttribute("model") d_search = rec.getAttribute("search") @@ -801,10 +802,30 @@ class DummyTagHandler: class MenuitemTagHandler: """Taghandler for the tag """ def __init__(self, master_handler): - pass + self.mh = master_handler def startElement(self, name, attributes): - return + + values = {} + + self.xml_id = attributes['id'] + + for attr in ('name', 'icon', 'sequence', 'parent_id', 'action'): + if attributes.get(attr): + values[attr] = attributes.get(attr).encode('utf8') + + + if values.get('parent_id') : + values['parent_id'] = self.mh.get_id(values['parent_id']) + + if values.get('action') : + type_attr = attributes.get('type', 'act_window').encode('utf8') + values['action'] = "ir.actions.%s,%d" %\ + (type_attr, self.mh.get_id(values['action'])) + + print values + self.values = values + def characters(self, data): pass @@ -814,6 +835,11 @@ class MenuitemTagHandler: if name != "menuitem": return self else: + + res = self.mh.pool.get('ir.model.data')._update( + self.mh.cursor, self.mh.user, + 'ir.ui.menu', self.mh.module, self.values, self.xml_id, + noupdate=self.mh.noupdate, mode=self.mh.mode) return None @@ -875,8 +901,7 @@ class RecordTagHandler: self.values[field_name] = answer[0]['id'] elif ref_attr: - # TODO avec cache sur les ids..: - #self.id_get(cursor, f_ref) + # TODO handle correctly the cache on ids self.values[field_name] = self.mh.get_id(ref_attr) elif eval_attr: From 1cb79819813603363b3bc20453c51e4314be91aa Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Thu, 3 Jan 2008 00:32:54 +0100 Subject: [PATCH 3/9] WIP, Sax api : removed some trailing whitespaces --- trytond/tools/convert.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index d4d95f4b1d..a03bdc31fb 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -12,7 +12,7 @@ import logging CDATA_START = re.compile('^\s*\<\!\[cdata\[', re.IGNORECASE) CDATA_END = re.compile('\]\]\>\s*$', re.IGNORECASE) - + def _ref(self, cursor): return lambda x: self.id_get(cursor, x) @@ -86,7 +86,7 @@ def _eval_xml(self, node, pool, cursor, user, idref, context=None): txt = CDATA_START.sub('', txt) txt = CDATA_END.sub('', txt) - return '\n' + txt + return '\n' + txt if f_type in ('char', 'int', 'float'): value = "" for child_node in node.childNodes: @@ -219,7 +219,7 @@ class XMLImport(object): if len(obj_id) > 64: Logger().notify_channel('init', LOG_ERROR, 'id: %s is to long (max: 64)' % xml_id) - + def _tag_delete(self, cursor, rec, data_node=None): d_model = rec.getAttribute("model") d_search = rec.getAttribute("search") @@ -814,7 +814,7 @@ class MenuitemTagHandler: if attributes.get(attr): values[attr] = attributes.get(attr).encode('utf8') - + if values.get('parent_id') : values['parent_id'] = self.mh.get_id(values['parent_id']) @@ -844,14 +844,14 @@ class MenuitemTagHandler: class RecordTagHandler: - + """Taghandler for the tag and all the tags inside it""" def __init__(self, master_handler): # Remind reference of parent handler self.mh = master_handler - + def startElement(self, name, attributes): # Manage the top level tag @@ -863,10 +863,10 @@ class RecordTagHandler: # create/update a dict containing fields values self.values = {} - + self.current_field = None self.cdata = False - + return self.xml_id # Manage included tags: @@ -891,8 +891,8 @@ class RecordTagHandler: cursor, self.mh.user, model.search(self.mh.cursor,self.mh.user, search_attr)) - if not answer: return - + if not answer: return + if field_name in model._columns: if model._columns[field_name]._type == 'many2many': self.values[field_name] = [(6, 0, [x['id'] for x in answer])] @@ -930,14 +930,14 @@ class RecordTagHandler: def characters(self, data): """If whe are in a field tag, consume all the content""" - + if not self.current_field: return # Escape start cdata tag if necessary if self.cdata == "start": data = CDATA_START.sub('', data) self.start_cdata = "inside" - + self.values[self.current_field] += data.encode('utf8') @@ -947,7 +947,7 @@ class RecordTagHandler: not 'record' we return self to keep our hand on the process. If name is 'record' we return None to end the delegation""" - + if name == "field": if not self.current_field: raise Exception("Application error" @@ -957,10 +957,10 @@ class RecordTagHandler: self.values[self.current_field] =\ CDATA_END.sub('', self.values[self.current_field]) self.cdata = 'done' - + self.current_field = None return self - + elif name == "record": # db access: TODO : use the object reference instead of # the name of the model because _update do a new get to @@ -988,7 +988,7 @@ class TrytondXmlHandler(sax.handler.ContentHandler): self.cursor = cursor self.user = 1 self.module = module - + # Tag handlders are used to delegate the processing self.taghandlerlist = { @@ -1038,11 +1038,11 @@ class TrytondXmlHandler(sax.handler.ContentHandler): return else: self.taghandler.startElement(name, attributes) - + def characters(self, data): if self.taghandler: self.taghandler.characters(data) - + def endElement(self, name): # Closing tag found, if we are in a delegation the handler @@ -1060,7 +1060,7 @@ def convert_xml_import_sax(cursor, module, xmlstream, idref=None, mode='init', parser = sax.make_parser() - # Tell the parser we are not interested in XML namespaces + # Tell the parser we are not interested in XML namespaces parser.setFeature(sax.handler.feature_namespaces, 0) handler = TrytondXmlHandler( @@ -1079,5 +1079,5 @@ def convert_xml_import_sax(cursor, module, xmlstream, idref=None, mode='init', return True -# use convert_xml_import_sax or convert_xml_import_dom +# use convert_xml_import_sax or convert_xml_import_dom convert_xml_import = convert_xml_import_sax From 45a7fbc4746b41cfb690089c333e46bbe595b63e Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Fri, 4 Jan 2008 00:06:13 +0100 Subject: [PATCH 4/9] * menuitem import : if a menuitem has no name, the name of the action is used. * updated xml files to use this feature instead of rely on the "string path" (ie name="parent/menu name") --- trytond/ir/actions.xml | 10 +++++----- trytond/ir/attachment.xml | 2 +- trytond/ir/cron.xml | 2 +- trytond/ir/exports.xml | 2 +- trytond/ir/ir.xml | 2 +- trytond/ir/model.xml | 6 +++--- trytond/ir/property.xml | 4 ++-- trytond/ir/rule.xml | 2 +- trytond/ir/sequence.xml | 2 +- trytond/ir/translation.xml | 2 +- trytond/ir/values.xml | 2 +- trytond/res/lang.xml | 2 +- trytond/res/request.xml | 2 +- trytond/res/res.xml | 2 +- trytond/res/users.xml | 8 ++++---- trytond/tools/convert.py | 17 +++++++++++------ trytond/workflow/workflow.xml | 12 ++++++------ 17 files changed, 42 insertions(+), 37 deletions(-) diff --git a/trytond/ir/actions.xml b/trytond/ir/actions.xml index 866f3e44fc..d5281c971c 100644 --- a/trytond/ir/actions.xml +++ b/trytond/ir/actions.xml @@ -1,7 +1,7 @@ - + ir.actions.actions.form ir.actions.actions @@ -58,7 +58,7 @@ - ir.actions.report.form @@ -110,7 +110,7 @@ - ir.actions.act_window.form @@ -182,7 +182,7 @@ - ir.actions.wizard.form @@ -234,7 +234,7 @@ - diff --git a/trytond/ir/attachment.xml b/trytond/ir/attachment.xml index c0987edc89..29257a5bc8 100644 --- a/trytond/ir/attachment.xml +++ b/trytond/ir/attachment.xml @@ -54,7 +54,7 @@ - diff --git a/trytond/ir/cron.xml b/trytond/ir/cron.xml index 61819a90eb..cc21058066 100644 --- a/trytond/ir/cron.xml +++ b/trytond/ir/cron.xml @@ -64,7 +64,7 @@ - diff --git a/trytond/ir/exports.xml b/trytond/ir/exports.xml index 4ad90c20ff..49be0a1ab2 100644 --- a/trytond/ir/exports.xml +++ b/trytond/ir/exports.xml @@ -48,7 +48,7 @@ - diff --git a/trytond/ir/ir.xml b/trytond/ir/ir.xml index 12ebe1a8bc..057da56bf9 100644 --- a/trytond/ir/ir.xml +++ b/trytond/ir/ir.xml @@ -2,6 +2,6 @@ - + diff --git a/trytond/ir/model.xml b/trytond/ir/model.xml index 59941a332e..e58ef27203 100644 --- a/trytond/ir/model.xml +++ b/trytond/ir/model.xml @@ -57,7 +57,7 @@ - ir.model.fields.form @@ -107,7 +107,7 @@ - ir.model.access.tree @@ -161,7 +161,7 @@ - diff --git a/trytond/ir/property.xml b/trytond/ir/property.xml index f4023d76e2..9f7a9ddfa5 100644 --- a/trytond/ir/property.xml +++ b/trytond/ir/property.xml @@ -53,7 +53,7 @@ - Properties @@ -75,7 +75,7 @@ - diff --git a/trytond/ir/rule.xml b/trytond/ir/rule.xml index fc1980d6a8..56ae16500d 100644 --- a/trytond/ir/rule.xml +++ b/trytond/ir/rule.xml @@ -55,7 +55,7 @@ - ir.rule.form diff --git a/trytond/ir/sequence.xml b/trytond/ir/sequence.xml index 3927633977..1dd52c1df9 100644 --- a/trytond/ir/sequence.xml +++ b/trytond/ir/sequence.xml @@ -1,7 +1,7 @@ - + ir.sequence.form ir.sequence diff --git a/trytond/ir/translation.xml b/trytond/ir/translation.xml index df5f58262c..7c4fd5bf3b 100644 --- a/trytond/ir/translation.xml +++ b/trytond/ir/translation.xml @@ -53,7 +53,7 @@ - diff --git a/trytond/ir/values.xml b/trytond/ir/values.xml index 7db70eb195..0d0a758b0b 100644 --- a/trytond/ir/values.xml +++ b/trytond/ir/values.xml @@ -57,7 +57,7 @@ - diff --git a/trytond/res/lang.xml b/trytond/res/lang.xml index b2b5a67dcf..7e800fca0f 100644 --- a/trytond/res/lang.xml +++ b/trytond/res/lang.xml @@ -34,7 +34,7 @@ - diff --git a/trytond/res/request.xml b/trytond/res/request.xml index 0fb25201ea..390c5c06ca 100644 --- a/trytond/res/request.xml +++ b/trytond/res/request.xml @@ -83,7 +83,7 @@ - res.request.link.form diff --git a/trytond/res/res.xml b/trytond/res/res.xml index df803941bb..34c4dfd13c 100644 --- a/trytond/res/res.xml +++ b/trytond/res/res.xml @@ -1,6 +1,6 @@ - + diff --git a/trytond/res/users.xml b/trytond/res/users.xml index 96de8437f5..fee4fcfc15 100644 --- a/trytond/res/users.xml +++ b/trytond/res/users.xml @@ -77,7 +77,7 @@ - res.group.form @@ -133,7 +133,7 @@ - res.role.form @@ -194,9 +194,9 @@ - - diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index a03bdc31fb..566b69eb89 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -770,13 +770,11 @@ def convert_xml_import_dom(cursor, module, xmlstream, idref=None, mode='init', #$$ # Notes: -# - classe monolythique vs instancier des sous-classe par type de record -# (que l'on re-reference a la volee des qu'on lit le debut du record). -# - utilser des pointeur de fct plutot que que inTitle=1 + # - analyser le code _update, source d'amelioration # - ajouter un handler d'exception # - faire un cache en debut de traitement : {xml_id: (db_id, model)} - +# - prevoir du cache sur les action pour booster la creation des menuitems # tuto : http://pyxml.sourceforge.net/topics/howto/node14.html from xml import sax @@ -820,10 +818,17 @@ class MenuitemTagHandler: if values.get('action') : type_attr = attributes.get('type', 'act_window').encode('utf8') + action_id = self.mh.get_id(values['action']) values['action'] = "ir.actions.%s,%d" %\ - (type_attr, self.mh.get_id(values['action'])) + (type_attr, action_id) + + + if not values.get('name'): + res = self.mh.pool.get('ir.actions.act_window').read( + self.mh.cursor, self.mh.user, action_id, ['name']) + values['name'] = res['name'] + - print values self.values = values diff --git a/trytond/workflow/workflow.xml b/trytond/workflow/workflow.xml index 07178f7c94..f007a1d763 100644 --- a/trytond/workflow/workflow.xml +++ b/trytond/workflow/workflow.xml @@ -1,7 +1,7 @@ - + workflow.form workflow @@ -51,7 +51,7 @@ - workflow.activity.form @@ -133,7 +133,7 @@ - workflow.transition.form @@ -190,7 +190,7 @@ - workflow.instance.form @@ -236,7 +236,7 @@ - workflow.workitem.form @@ -287,7 +287,7 @@ - From 2e0f0d06864ebd70b1d1d6eed1666f0adb36f275 Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Sat, 5 Jan 2008 02:12:11 +0100 Subject: [PATCH 5/9] Added icon resolution for menuitems, fixed some xml to comply with the spec. --- trytond/ir/module/category.xml | 2 +- trytond/ir/module/module.xml | 9 +++--- trytond/ir/module/repository.xml | 2 +- trytond/ir/sequence.xml | 4 +-- trytond/ir/ui/menu.xml | 2 +- trytond/ir/ui/ui.xml | 2 +- trytond/ir/ui/view.xml | 4 +-- trytond/tools/convert.py | 52 +++++++++++++++++++++++++++----- 8 files changed, 58 insertions(+), 19 deletions(-) diff --git a/trytond/ir/module/category.xml b/trytond/ir/module/category.xml index 426d289f94..101a56eee3 100644 --- a/trytond/ir/module/category.xml +++ b/trytond/ir/module/category.xml @@ -39,7 +39,7 @@ - diff --git a/trytond/ir/module/module.xml b/trytond/ir/module/module.xml index 182b3d7ae6..42da95fded 100644 --- a/trytond/ir/module/module.xml +++ b/trytond/ir/module/module.xml @@ -1,7 +1,7 @@ - + ir.module.module form @@ -95,7 +95,7 @@ tree_open ir.module.category,0 - Update Module List @@ -127,7 +127,7 @@ - @@ -183,7 +183,8 @@ ]]> - diff --git a/trytond/ir/module/repository.xml b/trytond/ir/module/repository.xml index 44b5b15314..664d85357f 100644 --- a/trytond/ir/module/repository.xml +++ b/trytond/ir/module/repository.xml @@ -50,7 +50,7 @@ - diff --git a/trytond/ir/sequence.xml b/trytond/ir/sequence.xml index 00eee893aa..c3fdeba0f8 100644 --- a/trytond/ir/sequence.xml +++ b/trytond/ir/sequence.xml @@ -60,7 +60,7 @@ - ir.sequence.type @@ -107,7 +107,7 @@ - diff --git a/trytond/ir/ui/menu.xml b/trytond/ir/ui/menu.xml index 6d16a4640d..16b966406f 100644 --- a/trytond/ir/ui/menu.xml +++ b/trytond/ir/ui/menu.xml @@ -75,7 +75,7 @@ - diff --git a/trytond/ir/ui/ui.xml b/trytond/ir/ui/ui.xml index 87b4cba54f..1e9bf9d053 100644 --- a/trytond/ir/ui/ui.xml +++ b/trytond/ir/ui/ui.xml @@ -1,6 +1,6 @@ - + diff --git a/trytond/ir/ui/view.xml b/trytond/ir/ui/view.xml index 32f78e757a..f64569404a 100644 --- a/trytond/ir/ui/view.xml +++ b/trytond/ir/ui/view.xml @@ -49,7 +49,7 @@ - ir.ui.view_sc @@ -96,7 +96,7 @@ - diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index 566b69eb89..fc2dd9332b 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -808,26 +808,64 @@ class MenuitemTagHandler: self.xml_id = attributes['id'] - for attr in ('name', 'icon', 'sequence', 'parent_id', 'action'): + for attr in ('name', 'icon', 'sequence', 'parent_id', 'action',): if attributes.get(attr): values[attr] = attributes.get(attr).encode('utf8') - if values.get('parent_id') : + if values.get('parent_id'): values['parent_id'] = self.mh.get_id(values['parent_id']) - if values.get('action') : + if values.get('action'): type_attr = attributes.get('type', 'act_window').encode('utf8') action_id = self.mh.get_id(values['action']) values['action'] = "ir.actions.%s,%d" %\ (type_attr, action_id) + #res = self.mh.pool.get('ir.actions.actions').read( + # self.mh.cursor, self.mh.user, action_id, ['name']) + + # TODO maybe use a prefetch for this: + self.mh.cursor.execute(""" + SELECT a.name, a.type, v.type, wv.view_mode + FROM ir_act_window a + left join ir_ui_view v on (v.id=a.view_id) + left join ir_act_window_view wv on (a.id=wv.act_window_id) + ORDER by wv.sequence + LIMIT 1 + """) + action_name, action_type, view_type, view_mode = \ + self.mh.cursor.fetchone() + + + # TODO the icon depends only on the action except for this + # damned 'type' attribute ! + menuitem_type = attributes.get('type','').encode('utf8') + availables_icons= { + "act_window": 'STOCK_NEW', + "report.xml": 'STOCK_PASTE', + "wizard": 'STOCK_EXECUTE', + "url": 'STOCK_JUMP_TO',} + + if menuitem_type in availables_icons: + values['icon'] = availables_icons[menuitem_type] + elif view_type == 'tree': + values['icon'] = 'STOCK_INDENT' + elif view_mode and view_mode.startswith('tree'): + values['icon'] = 'STOCK_JUSTIFY_FILL' + elif view_mode and view_mode.startswith('graph'): + values['icon'] = 'terp-graph' + elif view_mode and view_mode.startswith('calendar'): + values['icon'] = 'terp-calendar' + else: + values['icon'] = 'STOCK_NEW' if not values.get('name'): - res = self.mh.pool.get('ir.actions.act_window').read( - self.mh.cursor, self.mh.user, action_id, ['name']) - values['name'] = res['name'] - + if not values.get('action'): + raise Exception("Please provide at least a 'name' attributes " + "or a 'action' attributes on the menuitem tags.") + else: + values['name'] = action_name self.values = values From f757f186fce83c132e42de275ed8b1644ca64a3c Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Mon, 7 Jan 2008 03:13:59 +0100 Subject: [PATCH 6/9] Fixed non unique id in xml --- trytond/ir/property.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trytond/ir/property.xml b/trytond/ir/property.xml index 7e4dbefe19..2a46051bdd 100644 --- a/trytond/ir/property.xml +++ b/trytond/ir/property.xml @@ -52,7 +52,7 @@ + action="act_property_form_default" id="menu_property_form_default"/> Properties ir.actions.act_window From 4b7217d836d169043e10f82790b1c9c8a546184f Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Mon, 7 Jan 2008 03:14:40 +0100 Subject: [PATCH 7/9] Fix : added a test if write is called on a unique id --- trytond/ir/ui/view.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trytond/ir/ui/view.py b/trytond/ir/ui/view.py index 0e4cc23f68..4d94c4fbc3 100644 --- a/trytond/ir/ui/view.py +++ b/trytond/ir/ui/view.py @@ -65,6 +65,9 @@ class View(OSV): return res def write(self, cursor, user, ids, vals, context=None): + + if isinstance(ids, (int, long)): + ids = [ids] views = self.browse(cursor, user, ids) for view in views: # Restart the cache From 277e5e4460ba7f455848683400c83aa2258c5432 Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Mon, 7 Jan 2008 03:25:18 +0100 Subject: [PATCH 8/9] Function in ir.model.data rewriten to improve performance. --- trytond/ir/model.py | 319 ++++++++-------- trytond/module.py | 17 +- trytond/tools/convert.py | 801 +-------------------------------------- 3 files changed, 169 insertions(+), 968 deletions(-) diff --git a/trytond/ir/model.py b/trytond/ir/model.py index 718088f222..7059aa21d2 100644 --- a/trytond/ir/model.py +++ b/trytond/ir/model.py @@ -1,6 +1,6 @@ "model" from trytond.osv import fields, OSV -from trytond.netsvc import Logger, LocalService, LOG_ERROR, LOG_INFO +from trytond.netsvc import Logger, LocalService, LOG_ERROR, LOG_INFO, LOG_WARNING from trytond.osv.orm import except_orm from trytond.tools import Cache import time @@ -141,211 +141,182 @@ class ModelData(OSV): _name = 'ir.model.data' _description = __doc__ _columns = { - 'name': fields.char('XML Identifier', required=True, size=64), + 'fs_id': fields.char('Identifier on File System', required=True, + size=64, help="The id of the record as known on the file system."), 'model': fields.char('Model', required=True, size=64), 'module': fields.char('Module', required=True, size=64), - 'res_id': fields.integer('Resource ID'), - 'noupdate': fields.boolean('Non Updatable'), + 'db_id': fields.integer('Resource ID', + help="The id of the record in the database."), 'date_update': fields.datetime('Update Date'), - 'date_init': fields.datetime('Init Date') + 'date_init': fields.datetime('Init Date'), + 'values': fields.text('Values'), } _defaults = { 'date_init': lambda *a: time.strftime('%Y-%m-%d %H:%M:%S'), - 'date_update': lambda *a: time.strftime('%Y-%m-%d %H:%M:%S'), - 'noupdate': lambda *a: False } def __init__(self, pool): OSV.__init__(self, pool) - self.loads = {} - self.doinit = True - self.unlink_mark = {} - def _get_id(self, cursor, user, module, xml_id): - ids = self.search(cursor, user, [ - ('module', '=', module), - ('name', '=', xml_id), - ]) - assert len(ids)==1, '%d reference(s) to %s. ' \ - 'You should have only one !' % (len(ids),xml_id) - return ids[0] + # fs2db contain the data of the table ir.model.data. This + # fs2db provide a mapping from the data in the fs to the data + # in the db. This mean that this dict can become very big if a + # lot of files are imported. (improvement store a hash instead of all the values). + self.fs2db = None + # imported_records keep tracks of all the records that have + # been imported. + self.imported_records = {} - _get_id = Cache()(_get_id) - def _update_dummy(self, cursor, user, model, module, xml_id=False): - if not xml_id: - return False - try: - obj_id = self.read(cursor, user, self._get_id(cursor, user, module, - xml_id), ['res_id'])['res_id'] - self.loads[(module, xml_id)] = (model, obj_id) - except: - obj_id = False - return obj_id + def populate_fs2db(self, cursor, user): + """Fetch all the table in one shot (if necessary).""" + if not self.fs2db: self.fs2db = {} + if cursor.dbname not in self.fs2db: + module_data_ids = self.search(cursor, user,[]) + self.fs2db[cursor.dbname] = dict([ + ((x.fs_id, x.module), + (x.db_id, x.model, x.values)) for x in \ + self.browse(cursor, user, module_data_ids)]) - def _update(self, cursor, user, model, module, values, xml_id='', - noupdate=False, mode='init', res_id=False): - model_obj = self.pool.get(model) - if xml_id and ('.' in xml_id): - assert len(xml_id.split('.')) == 2, '"%s" contains too many dots. '\ - 'XML ids should not contain dots ! ' \ + def get_id(self, cursor, user, module, fs_id): + + # Pre-Fetch all the data at once + self.populate_fs2db(cursor, user) + + # return only the db_id + if (fs_id, module) in self.fs2db[cursor.dbname]: + return self.fs2db[cursor.dbname][(fs_id, module)][0] + else: + raise Exception("Reference to %s not found"% ".".join([module,fs_id])) + + def import_record(self, cursor, user, model, module, values, fs_id): + + if not fs_id: + raise Exception('import_record : Argument fs_id is mandatory') + + if '.' in fs_id: + assert len(fs_id.split('.')) == 2, '"%s" contains too many dots. '\ + 'file system ids should contain ot most one dot ! ' \ 'These are used to refer to other modules data, ' \ - 'as in module.reference_id' % (xml_id) - module, xml_id = xml_id.split('.') - if (not xml_id) and (not self.doinit): - return False - action_id = False - if xml_id: - cursor.execute('SELECT id, res_id FROM ir_model_data ' \ - 'WHERE module = %s AND name = %s', (module,xml_id)) - results = cursor.fetchall() - for action_id2, res_id2 in results: - cursor.execute('SELECT id ' \ - 'FROM ' + self.pool.get(model)._table + ' ' \ - 'WHERE id = %d', (res_id2,)) - result3 = cursor.fetchone() - if not result3: - cursor.execute('DELETE FROM ir_model_data ' \ - 'WHERE id = %d', (action_id2,)) + 'as in module.reference_id' % (fs_id) + + module, fs_id = fs_id.split('.') + + self.populate_fs2db(cursor, user) + + # keep track of the lits of parsed records. + if cursor.dbname in self.imported_records: + self.imported_records[cursor.dbname].append((fs_id, module)) + else: + self.imported_records[cursor.dbname]= [(fs_id, module)] + + if (fs_id, module) in self.fs2db[cursor.dbname]: + # this record is already in the db: + db_id, db_model, old_values = \ + self.fs2db[cursor.dbname][(fs_id, module)] + old_values = eval(old_values) + + # Check if this record has been modified in the db: + if model != db_model: + raise Exception("This record try to overwrite" + "data with the wrong model.") + object_ref = self.pool.get(model) + db_values = object_ref.read(cursor, user, db_id, old_values.keys()) + + modified = False + for key in old_values: + + if object_ref._columns[key]._type == 'many2one': + db_field = db_values[key][0] + elif object_ref._columns[key]._type in ['one2one', 'one2many', "many2many"]: + logger = Logger() + logger.notify_channel('init', LOG_WARNING, + 'Field %s on %s : integrity not tested.'%(key, model)) + continue else: - res_id, action_id = res_id2, action_id2 + db_field = db_values[key] - if action_id and res_id: - model_obj.write(cursor, user, [res_id], values) - self.write(cursor, user, [action_id], { - 'date_update': time.strftime('%Y-%m-%d %H:%M:%S'), - }) - elif res_id: - model_obj.write(cursor, user, [res_id], values) - if xml_id: + if old_values[key] != db_field: + modified = True + break + + if not modified: + # Update the db with the values + # TODO : check if this id exist for this model. + + self.pool.get(model).write(cursor, user, db_id, values) self.create(cursor, user, { - 'name': xml_id, + 'fs_id': fs_id, 'model': model, - 'module':module, - 'res_id':res_id, - 'noupdate': noupdate, + 'module': module, + 'db_id': db_id, + 'values': str(values), + 'date_update': time.strftime('%Y-%m-%d %H:%M:%S'), }) - if model_obj._inherits: - for table in model_obj._inherits: - inherit_id = model_obj.browse(cursor, user, - res_id)[model_obj._inherits[table]] - self.create(cursor, user, { - 'name': xml_id + '_' + table.replace('.', '_'), - 'model': table, - 'module': module, - 'res_id': inherit_id, - 'noupdate': noupdate, - }) + + # Update fs2db: + self.fs2db[cursor.dbname][(fs_id, module)]= (db_id, model, str(values)) + else: + # We may not overwrite modified data: + logger = Logger() + logger.notify_channel('init', LOG_INFO, + 'Record %s ignored, the corresponding data (%s@%s) ' + 'in the db has been modified' % (fs_id,db_id, model)) + else: - if mode == 'init' or (mode == 'update' and xml_id): - res_id = model_obj.create(cursor, user, values) - if xml_id: - self.create(cursor, user, { - 'name': xml_id, - 'model': model, - 'module': module, - 'res_id': res_id, - 'noupdate': noupdate - }) - if model_obj._inherits: - for table in model_obj._inherits: - inherit_id = model_obj.browse(cursor, user, - res_id)[model_obj._inherits[table]] - self.create(cursor, user, { - 'name': xml_id + '_' + table.replace('.', '_'), - 'model': table, - 'module': module, - 'res_id': inherit_id, - 'noupdate': noupdate, - }) - if xml_id: - if res_id: - self.loads[(module, xml_id)] = (model, res_id) - return res_id + # this record is new + db_id = self.pool.get(model).create(cursor, user, values) + self.create(cursor, user, { + 'fs_id': fs_id, + 'model': model, + 'module': module, + 'db_id': db_id, + 'values': str(values), + }) + # update fs2db: + self.fs2db[cursor.dbname][(fs_id, module)]= (db_id, model, str(values)) - def _unlink(self, cursor, user, model, ids, direct=False): - #self.pool.get(model).unlink(cursor, user, ids) - for obj_id in ids: - self.unlink_mark[(model, obj_id)]=False - cursor.execute('DELETE FROM ir_model_data ' \ - 'WHERE res_id = %d AND model = %s', (obj_id, model)) - return True + def post_import(self, cursor, user, modules): - def ir_set(self, cursor, user, key, key2, name, models, value, - replace=True, isobject=False, meta=None, xml_id=False): - obj = self.pool.get('ir.values') - if type(models[0])==type([]) or type(models[0])==type(()): - model, res_id = models[0] - else: - res_id = None - model = models[0] - - if res_id: - where = ' AND res_id = %d' % (res_id,) - else: - where = ' AND (res_id IS NULL)' - - if key2: - where += ' AND key2 = \'%s\'' % (key2,) - else: - where += ' AND (key2 IS NULL)' - - cursor.execute('SELECT * FROM ir_values ' \ - 'WHERE model = %s AND key = %s AND name = %s' + where, - (model, key, name)) - res = cursor.fetchone() - if not res: - res = obj.set(cursor, user, key, key2, name, models, value, - replace, isobject, meta) - elif xml_id: - cursor.execute('UPDATE ir_values SET value = %s ' \ - 'WHERE model = %s AND key = %s AND name = %s' + where, - (value, model, key, name)) - return True - - def _process_end(self, cursor, user, modules): - if not modules: + # Test because of a wrong extra call see todo at the end of + # load_module_graph in module.py + # Globaly this function is a bit dirty. + if not (self.fs2db and self.fs2db.get(cursor.dbname)): return True - module_str = ["'%s'" % m for m in modules] - cursor.execute('SELECT id, name, model, res_id, module ' \ - 'FROM ir_model_data ' \ - 'WHERE module IN (' + ','.join(module_str) + ') ' \ - 'AND NOT noupdate') - wkf_todo = [] - for (obj_id, name, model, res_id, module) in cursor.fetchall(): - if (module, name) not in self.loads: - self.unlink_mark[(model, res_id)] = obj_id + + wf_service = LocalService("workflow") + + data_unlink = [] + + for ((fs_id, module),(db_id, model, values)) in \ + self.fs2db[cursor.dbname].items(): + + if module in modules and \ + ((fs_id, module) not in self.imported_records[cursor.dbname]): + if model == 'workflow.activity': - cursor.execute('SELECT res_type, res_id ' \ + cursor.execute('SELECT res_type, db_id ' \ 'FROM wkf_instance ' \ 'WHERE id IN (' \ 'SELECT inst_id FROM wkf_workitem ' \ - 'WHERE act_id = %d)', (res_id,)) + 'WHERE act_id = %d)', (db_id,)) wkf_todo.extend(cursor.fetchall()) cursor.execute("UPDATE wkf_transition " \ "SET condition = 'True', role_id = NULL, " \ "signal = NULL, act_to = act_from, " \ "act_from = %d " \ - "WHERE act_to = %d", (res_id, res_id)) + "WHERE act_to = %d", (db_id, db_id)) cursor.execute("DELETE FROM wkf_transition " \ - "WHERE act_to = %d", (res_id,)) + "WHERE act_to = %d", (db_id,)) - for model, obj_id in wkf_todo: - wf_service = LocalService("workflow") - wf_service.trg_write(user, model, obj_id, cursor) + wf_service.trg_write(user, model, db_id, cursor) - for (model, obj_id) in self.unlink_mark.keys(): - if self.pool.get(model): logger = Logger() logger.notify_channel('init', LOG_INFO, - 'Deleting %s@%s' % (obj_id, model)) + 'Deleting %s@%s' % (db_id, model)) try: - self.pool.get(model).unlink(cursor, user, [obj_id]) - if self.unlink_mark[(model, obj_id)]: - self.unlink(cursor, user, - [self.unlink_mark[(model, obj_id)]]) - cursor.execute('DELETE FROM ir_values WHERE value=%s', - (model + ',' + str(obj_id),)) + self.pool.get(model).unlink(cursor, user, db_id) + data_unlink.append((fs_id, module)) except: logger.notify_channel('init', LOG_ERROR, 'Could not delete id: %d of model %s\t' \ @@ -353,7 +324,19 @@ class ModelData(OSV): 'that points to this resource\t' \ 'You should manually fix this ' \ 'and restart --update=module' % \ - (obj_id, model)) + (db_id, model)) + + if data_unlink: + cursor.execute(""" + DELETE FROM ir_model_data + WHERE (fs_id, module) in (%s) + """% ','.join(["('%s','%s')"%x for x in data_unlink])) + + del self.fs2db[cursor.dbname] + del self.imported_records[cursor.dbname] + return True + + ModelData() diff --git a/trytond/module.py b/trytond/module.py index c152e90638..5f9729799f 100644 --- a/trytond/module.py +++ b/trytond/module.py @@ -158,8 +158,8 @@ def init_module_objects(cursor, module_name, obj_list): for obj in obj_list: obj.auto_init(cursor) -def load_module_graph(cursor, graph, **kwargs): - # **kwargs is passed directly to convert_xml_import +def load_module_graph(cursor, graph): + package_todo = [] statusi = 0 for package in graph: @@ -203,8 +203,7 @@ def load_module_graph(cursor, graph, **kwargs): cursor.execute(new_query) else: tools.convert_xml_import(cursor, module, - tools.file_open(OPJ(module, filename)), - idref, mode=mode, demo=demo, **kwargs) + tools.file_open(OPJ(module, filename))) if demo: cursor.execute('UPDATE ir_module_module SET demo = %s ' \ 'WHERE name = %s', (True, package.name)) @@ -216,7 +215,8 @@ def load_module_graph(cursor, graph, **kwargs): statusi += 1 pool = pooler.get_pool(cursor.dbname) - pool.get('ir.model.data')._process_end(cursor, 1, package_todo) + # TODO : post_import is called even if there not init nor update + pool.get('ir.model.data').post_import(cursor, 1, package_todo) cursor.commit() def register_classes(): @@ -263,10 +263,9 @@ def load_modules(database, force_demo=False, update_module=False): "WHERE state IN ('installed', 'to upgrade', 'to remove')") module_list = [name for (name,) in cursor.fetchall()] graph = create_graph(module_list, force) - report = tools.AssertionReport() - load_module_graph(cursor, graph, report=report) - if report.get_report(): - Logger().notify_channel('init', LOG_INFO, 'assert:%s' % report) + + load_module_graph(cursor, graph) + for kind in ('init', 'update'): CONFIG[kind] = {} diff --git a/trytond/tools/convert.py b/trytond/tools/convert.py index fc2dd9332b..c16508d45e 100644 --- a/trytond/tools/convert.py +++ b/trytond/tools/convert.py @@ -13,769 +13,9 @@ import logging CDATA_START = re.compile('^\s*\<\!\[cdata\[', re.IGNORECASE) CDATA_END = re.compile('\]\]\>\s*$', re.IGNORECASE) -def _ref(self, cursor): - return lambda x: self.id_get(cursor, x) +# TODO : +# Add some exception helper, and give a good diagnostic for the errors -def _obj(pool, cursor, user, model_str, context=None): - model = pool.get(model_str) - return lambda x: model.browse(cursor, user, x, context=context) - -def _eval_xml(self, node, pool, cursor, user, idref, context=None): - if context is None: - context = {} - if node.nodeType == node.TEXT_NODE: - return node.data.encode("utf8") - elif node.nodeType == node.ELEMENT_NODE: - if node.nodeName in ('field','value'): - f_type = node.getAttribute('type') or 'char' - f_model = node.getAttribute("model").encode('ascii') - if len(node.getAttribute('search')): - f_search = node.getAttribute("search").encode('utf-8') - f_use = node.getAttribute("use").encode('ascii') - f_name = node.getAttribute("name").encode('utf-8') - if len(f_use)==0: - f_use = "id" - args = eval(f_search, idref) - ids = pool.get(f_model).search(cursor, user, args) - if f_use != 'id': - ids = [x[f_use] for x in pool.get(f_model).read( - cursor, user, ids, [f_use])] - _cols = pool.get(f_model)._columns - if (f_name in _cols) and _cols[f_name]._type=='many2many': - return ids - f_val = False - if len(ids): - f_val = ids[0] - if isinstance(f_val, tuple): - f_val = f_val[0] - return f_val - a_eval = node.getAttribute('eval') - if len(a_eval): - import time - idref['time'] = time - idref['version'] = VERSION.rsplit('.', 1)[0] - idref['ref'] = lambda x: self.id_get(cursor, x) - if len(f_model): - idref['obj'] = _obj(self.pool, cursor, user, f_model, - context=context) - try: - import pytz - except: - Logger().notify_channel("init", LOG_INFO, - 'could not find pytz library') - - - class Pytz(object): - all_timezones = [] - - pytz = Pytz() - idref['pytz'] = pytz - return eval(a_eval, idref) - if f_type == 'xml': - - def _process(string, idref): - matches = re.findall('[^%]%\((.*?)\)[ds]', string) - for i in matches: - if not i in idref: - idref[i] = self.id_get(cursor, i) - return string % idref - - txt = _process("".join([x.toxml().encode("utf8") \ - for x in node.childNodes]), idref) - - txt = CDATA_START.sub('', txt) - txt = CDATA_END.sub('', txt) - - return '\n' + txt - if f_type in ('char', 'int', 'float'): - value = "" - for child_node in node.childNodes: - value += str(_eval_xml(self, child_node, pool, cursor, - user, idref)) - if f_type == 'int': - value = value.strip() - if value == 'None': - return None - else: - value = int(value.strip()) - elif f_type == 'float': - value = float(value.strip()) - return value - elif f_type in ('list', 'tuple'): - res = [] - for child_node in node.childNodes: - if child_node.nodeType != child_node.ELEMENT_NODE \ - or child_node.nodeName != 'value': - continue - res.append(_eval_xml(self, child_node, pool, cursor, - user, idref)) - if f_type == 'tuple': - return tuple(res) - return res - elif node.nodeName=="getitem": - for child_node in node.childNodes: - if child_node.nodeType != child_node.ELEMENT_NODE: - continue - res = _eval_xml(self, child_node, pool, cursor, user, idref) - if not res: - raise LookupError - elif node.getAttribute('type') in ("int", "list"): - return res[int(node.getAttribute('index'))] - else: - return res[node.getAttribute('index').encode("utf8")] - elif node.nodeName=="function": - args = [] - a_eval = node.getAttribute('eval') - if len(a_eval): - idref['ref'] = lambda x: self.id_get(cursor, x) - args = eval(a_eval, idref) - for child_node in node.childNodes: - if child_node.nodeType != child_node.ELEMENT_NODE: - continue - args.append(_eval_xml(self, child_node, pool, cursor, user, - idref, context)) - model = pool.get(node.getAttribute('model')) - method = node.getAttribute('name') - res = getattr(model, method)(cursor, user, *args) - return res - elif node.nodeName=="test": - value = "" - for child_node in node.childNodes: - value += str(_eval_xml(self, child_node, pool, cursor, user, - idref, context=context)) - return value - -ESCAPE_RE = re.compile(r'(? 64: - Logger().notify_channel('init', LOG_ERROR, - 'id: %s is to long (max: 64)' % xml_id) - - def _tag_delete(self, cursor, rec, data_node=None): - d_model = rec.getAttribute("model") - d_search = rec.getAttribute("search") - d_id = rec.getAttribute("id") - ids = [] - if len(d_search): - ids = self.pool.get(d_model).search(cursor, self.user, - eval(d_search)) - if len(d_id): - ids.append(self.id_get(cursor, d_id)) - if len(ids): - self.pool.get(d_model).unlink(cursor, self.user, ids) - #self.pool.get('ir.model.data')._unlink(cursor, self.user, d_model, - # ids, direct=True) - return False - - def _tag_report(self, cursor, rec, data_node=None): - res = {} - for dest, attr in ( - ('name', 'string'), - ('model', 'model'), - ('report_name', 'name'), - ): - res[dest] = rec.getAttribute(attr).encode('utf8') - assert res[dest], "Attribute %s of report is empty !" % (attr,) - for field, dest in ( - ('rml', 'report_rml'), - ('xml', 'report_xml'), - ('xsl', 'report_xsl'), - ): - if rec.hasAttribute(field): - res[dest] = rec.getAttribute(field).encode('utf8') - if rec.hasAttribute('auto'): - res['auto'] = eval(rec.getAttribute('auto')) - if rec.hasAttribute('header'): - res['header'] = eval(rec.getAttribute('header')) - res['multi'] = rec.hasAttribute('multi') \ - and eval(rec.getAttribute('multi')) - xml_id = rec.getAttribute('id').encode('utf8') - self._test_xml_id(xml_id) - obj_id = self.pool.get('ir.model.data')._update(cursor, self.user, - "ir.actions.report.xml", self.module, res, xml_id, - mode=self.mode) - self.idref[xml_id] = int(obj_id) - if not rec.hasAttribute('menu') or eval(rec.getAttribute('menu')): - keyword = str(rec.getAttribute('keyword') or 'client_print_multi') - value = 'ir.actions.report.xml,' + str(obj_id) - replace = rec.hasAttribute('replace') \ - and rec.getAttribute("replace") - self.pool.get('ir.model.data').ir_set(cursor, self.user, 'action', - keyword, res['name'], [res['model']], value, - replace=replace, isobject=True, xml_id=xml_id) - return False - - def _tag_function(self, cursor, rec, data_node=None): - if self.isnoupdate(data_node) and self.mode != 'init': - return - context = self.get_context(data_node, rec, {'ref': _ref(self, cursor)}) - user = self.get_uid(cursor, self.user, data_node, rec) - _eval_xml(self, rec, self.pool, cursor, user, self.idref, - context=context) - return False - - def _tag_wizard(self, cursor, rec, data_node=None): - string = rec.getAttribute("string").encode('utf8') - model = rec.getAttribute("model").encode('utf8') - name = rec.getAttribute("name").encode('utf8') - xml_id = rec.getAttribute('id').encode('utf8') - self._test_xml_id(xml_id) - multi = rec.hasAttribute('multi') and eval(rec.getAttribute('multi')) - res = {'name': string, 'wiz_name': name, 'multi':multi} - - obj_id = self.pool.get('ir.model.data')._update(cursor, self.user, - "ir.actions.wizard", self.module, res, xml_id, mode=self.mode) - self.idref[xml_id] = int(obj_id) - # ir_set - if (not rec.hasAttribute('menu') or eval(rec.getAttribute('menu'))) \ - and obj_id: - keyword = str(rec.getAttribute('keyword') or 'client_action_multi') - value = 'ir.actions.wizard,' + str(obj_id) - replace = rec.hasAttribute('replace') and \ - rec.getAttribute("replace") or True - self.pool.get('ir.model.data').ir_set(cursor, self.user, 'action', - keyword, string, [model], value, replace=replace, - isobject=True, xml_id=xml_id) - return False - - def _tag_act_window(self, cursor, rec, data_node=None): - name = rec.hasAttribute('name') \ - and rec.getAttribute('name').encode('utf-8') - xml_id = rec.getAttribute('id').encode('utf8') - self._test_xml_id(xml_id) - ftype = rec.hasAttribute('type') \ - and rec.getAttribute('type').encode('utf-8') \ - or 'ir.actions.act_window' - view_id = False - if rec.hasAttribute('view'): - view_id = self.id_get(cursor, - rec.getAttribute('view').encode('utf-8')) - domain = rec.hasAttribute('domain') \ - and rec.getAttribute('domain').encode('utf-8') - context = rec.hasAttribute('context') \ - and rec.getAttribute('context').encode('utf-8') \ - or '{}' - res_model = rec.getAttribute('res_model').encode('utf-8') - src_model = rec.hasAttribute('src_model') \ - and rec.getAttribute('src_model').encode('utf-8') - view_type = rec.hasAttribute('view_type') \ - and rec.getAttribute('view_type').encode('utf-8') \ - or 'form' - view_mode = rec.hasAttribute('view_mode') \ - and rec.getAttribute('view_mode').encode('utf-8') \ - or 'tree,form' - usage = rec.hasAttribute('usage') \ - and rec.getAttribute('usage').encode('utf-8') - limit = rec.hasAttribute('limit') \ - and rec.getAttribute('limit').encode('utf-8') - auto_refresh = rec.hasAttribute('auto_refresh') \ - and rec.getAttribute('auto_refresh').encode('utf-8') - - res = { - 'name': name, - 'type': ftype, - 'view_id': view_id, - 'domain': domain, - 'context': context, - 'res_model': res_model, - 'src_model': src_model, - 'view_type': view_type, - 'view_mode': view_mode, - 'usage': usage, - 'limit': limit, - 'auto_refresh': auto_refresh, - } - - obj_id = self.pool.get('ir.model.data')._update(cursor, self.user, - 'ir.actions.act_window', self.module, res, xml_id, - mode=self.mode) - self.idref[xml_id] = int(obj_id) - - if src_model: - keyword = 'client_action_relate' - value = 'ir.actions.act_window,' + str(obj_id) - replace = rec.hasAttribute('replace') \ - and rec.getAttribute('replace') - self.pool.get('ir.model.data').ir_set(cursor, self.user, 'action', - keyword, xml_id, [src_model], value, replace=replace, - isobject=True, xml_id=xml_id) - # TODO add remove ir.model.data - return False - - def _tag_ir_set(self, cursor, rec, data_node=None): - if not self.mode == 'init': - return False - res = {} - for field in [i for i in rec.childNodes \ - if (i.nodeType == i.ELEMENT_NODE and i.nodeName=="field")]: - f_name = field.getAttribute("name").encode('utf-8') - f_val = _eval_xml(self, field, self.pool, cursor, self.user, - self.idref) - res[f_name] = f_val - self.pool.get('ir.model.data').ir_set(cursor, self.user, res['key'], - res['key2'], res['name'], res['models'], res['value'], - replace=res.get('replace',True), - isobject=res.get('isobject', False), meta=res.get('meta',None)) - return False - - def _tag_workflow(self, cursor, rec, data_node=None): - if self.isnoupdate(data_node) and self.mode != 'init': - return - model = str(rec.getAttribute('model')) - w_ref = rec.getAttribute('ref') - if len(w_ref): - obj_id = self.id_get(cursor, w_ref) - else: - assert rec.childNodes, 'You must define a child node ' \ - 'if you dont give a ref' - element_childs = [i for i in rec.childNodes \ - if i.nodeType == i.ELEMENT_NODE] - assert len(element_childs) == 1, 'Only one child node ' \ - 'is accepted (%d given)' % len(rec.childNodes) - obj_id = _eval_xml(self, element_childs[0], self.pool, cursor, - self.user, self.idref) - - user = self.get_uid(cursor, self.user, data_node, rec) - wf_service = LocalService("workflow") - wf_service.trg_validate(user, model, - obj_id, - str(rec.getAttribute('action')), cursor) - return False - - def _tag_menuitem(self, cursor, rec, data_node=None): - rec_id = rec.getAttribute("id").encode('ascii') - assert rec_id, "No id on menutiem %s" % \ - rec.getAttribute("name").encode('utf8') - self._test_xml_id(rec_id) - m_l = [escape(x) for x in ESCAPE_RE.split( - rec.getAttribute("name").encode('utf8'))] - pid = False - for idx, menu_elem in enumerate(m_l): - if pid: - cursor.execute('SELECT id FROM ir_ui_menu ' \ - 'WHERE parent_id = %d AND name = %s', - (pid, menu_elem)) - else: - cursor.execute('SELECT id FROM ir_ui_menu ' \ - 'WHERE parent_id IS NULL AND name = %s', - (menu_elem,)) - res = cursor.fetchone() - if idx == (len(m_l) - 1): - # we are at the last menu element/level (it's a leaf) - values = {'parent_id': pid, 'name': menu_elem} - - if rec.hasAttribute('action'): - a_action = rec.getAttribute('action').encode('utf8') - a_type = rec.getAttribute('type').encode('utf8') \ - or 'act_window' - icons = { - "act_window": 'STOCK_NEW', - "report.xml": 'STOCK_PASTE', - "wizard": 'STOCK_EXECUTE', - "url": 'STOCK_JUMP_TO' - } - values['icon'] = icons.get(a_type,'STOCK_NEW') - if a_type == 'act_window': - a_id = self.id_get(cursor, a_action) - cursor.execute('SELECT view_type, ' \ - 'name, view_id ' \ - 'FROM ir_act_window ' \ - 'WHERE id = %d', (int(a_id),)) - action_type, action_name, view_id = \ - cursor.fetchone() - if view_id: - cursor.execute('SELECT type FROM ir_ui_view ' \ - 'WHERE id = %d', (int(view_id),)) - action_mode, = cursor.fetchone() - cursor.execute('SELECT view_mode ' \ - 'FROM ir_act_window_view ' \ - 'WHERE act_window_id = %d ' \ - 'ORDER BY sequence LIMIT 1', (int(a_id),)) - if cursor.rowcount: - action_mode, = cursor.fetchone() - if action_type == 'tree': - values['icon'] = 'STOCK_INDENT' - elif action_mode and action_mode.startswith('tree'): - values['icon'] = 'STOCK_JUSTIFY_FILL' - elif action_mode and action_mode.startswith('graph'): - values['icon'] = 'terp-graph' - elif action_mode and action_mode.startswith('calendar'): - values['icon'] = 'terp-calendar' - if not values['name']: - values['name'] = action_name - if rec.hasAttribute('sequence'): - values['sequence'] = int(rec.getAttribute('sequence')) - if rec.hasAttribute('icon'): - values['icon'] = str(rec.getAttribute('icon')) - if rec.hasAttribute('groups'): - g_names = rec.getAttribute('groups').split(',') - groups_value = [] - groups_obj = self.pool.get('res.group') - for group in g_names: - if group.startswith('-'): - group_id = self.id_get(cursor, group[1:]) - groups_value.append((3, group_id)) - else: - group_id = self.id_get(cursor, group) - groups_value.append((4, group_id)) - values['groups_id'] = groups_value - if rec.hasAttribute('action'): - a_action = rec.getAttribute('action').encode('utf8') - a_type = rec.getAttribute('type').encode('utf8') \ - or 'act_window' - a_id = self.id_get(cursor, a_action) - values['action'] = "ir.actions.%s,%d" % (a_type, a_id) - xml_id = rec.getAttribute('id').encode('utf8') - self._test_xml_id(xml_id) - pid = self.pool.get('ir.model.data')._update(cursor, self.user, - 'ir.ui.menu', self.module, values, xml_id, - mode=self.mode, res_id=(res and res[0] or False)) - else: - assert res, "The parent menuitem %s does not exist!" % \ - (menu_elem,) - pid = res[0] - self.idref[rec_id] = int(pid) - return ('ir.ui.menu', pid) - - def _assert_equals(self, i, j, prec=4): - return not round(i - j, prec) - - def _tag_assert(self, cursor, rec, data_node=None): - if self.isnoupdate(data_node) and self.mode != 'init': - return - - rec_model = rec.getAttribute("model").encode('ascii') - model = self.pool.get(rec_model) - assert model, "The model %s does not exist !" % (rec_model,) - rec_id = rec.getAttribute("id").encode('ascii') - self._test_xml_id(rec_id) - rec_src = rec.getAttribute("search").encode('utf8') - rec_src_count = rec.getAttribute("count") - - severity = rec.getAttribute("severity").encode('ascii') or 'info' - - rec_string = rec.getAttribute("string").encode('utf8') or 'unknown' - - ids = None - eval_dict = {'ref': _ref(self, cursor)} - context = self.get_context(data_node, rec, eval_dict) - user = self.get_uid(cursor, self.user, data_node, rec) - if len(rec_id): - ids = [self.id_get(cursor, rec_id)] - elif len(rec_src): - args = eval(rec_src, eval_dict) - ids = self.pool.get(rec_model).search(cursor, user, args, - context=context) - if len(rec_src_count): - count = int(rec_src_count) - if len(ids) != count: - self.assert_report.record_assertion(False, severity) - Logger().notify_channel('init', severity, - 'assertion "' + rec_string + \ - '" failed ! (search count is incorrect: ' \ - + str(len(ids)) + ')' ) - sevval = getattr(logging, severity.upper()) - if sevval > CONFIG['assert_exit_level']: - # TODO: define a dedicated exception - raise Exception('Severe assertion failure') - return - - assert ids != None, 'You must give either an id or a search criteria' - - ref = _ref(self, cursor) - for brrec in model.browse(cursor, user, ids, context): - - - class Dict(dict): - - def __getitem__(self, key): - if key in brrec: - return brrec[key] - return dict.__getitem__(self, key) - - eval_globals = Dict() - eval_globals['floatEqual'] = self._assert_equals - eval_globals['ref'] = ref - eval_globals['_ref'] = ref - for test in [i for i in rec.childNodes \ - if (i.nodeType == i.ELEMENT_NODE and i.nodeName=="test")]: - f_expr = test.getAttribute("expr").encode('utf-8') - f_val = _eval_xml(self, test, self.pool, cursor, user, - self.idref, context=context) or True - if eval(f_expr, eval_globals) != f_val: # assertion failed - self.assert_report.record_assertion(False, severity) - Logger().notify_channel('init', severity, - 'assertion "' + rec_string + \ - '" failed ! (tag ' + test.toxml() + ')' ) - sevval = getattr(logging, severity.upper()) - if sevval > CONFIG['assert_exit_level']: - # TODO: define a dedicated exception - raise Exception('Severe assertion failure') - return - else: # all tests were successful for this assertion tag (no break) - self.assert_report.record_assertion(True, severity) - - def _tag_record(self, cursor, rec, data_node=None): - rec_model = rec.getAttribute("model").encode('ascii') - model = self.pool.get(rec_model) - assert model, "The model %s does not exist !" % (rec_model,) - rec_id = rec.getAttribute("id").encode('ascii') - self._test_xml_id(rec_id) - -# if not rec_id and not self.isnoupdate(data_node): -# print "Warning", rec_model - - if self.isnoupdate(data_node) and not self.mode == 'init': - # check if the xml record has an id string - if rec_id: - obj_id = self.pool.get('ir.model.data')._update_dummy(cursor, - self.user, rec_model, self.module, rec_id) - # check if the resource already existed at the last update - if obj_id: - # if it existed, we don't update the data, but we need to - # know the id of the existing record anyway - self.idref[rec_id] = int(obj_id) - return None - else: - # if the resource didn't exist - if rec.getAttribute("forcecreate"): - # we want to create it, so we let the normal - # "update" behavior happen - pass - else: - # otherwise do nothing - return None - else: - # otherwise it is skipped - return None - - res = {} - for field in [i for i in rec.childNodes \ - if (i.nodeType == i.ELEMENT_NODE and i.nodeName=="field")]: - #TODO: most of this code is duplicated above (in _eval_xml) - f_name = field.getAttribute("name").encode('utf-8') - f_ref = field.getAttribute("ref").encode('ascii') - f_search = field.getAttribute("search").encode('utf-8') - f_model = field.getAttribute("model").encode('ascii') - if not f_model and model._columns.get(f_name, False): - f_model = model._columns[f_name]._obj - f_use = field.getAttribute("use").encode('ascii') or 'id' - f_val = False - - if len(f_search): - args = eval(f_search, self.idref) - field = [] - assert f_model, 'Define an attribute ' \ - 'model="..." in your .XML file !' - f_obj = self.pool.get(f_model) - # browse the objects searched - objs = f_obj.browse(cursor, self.user, f_obj.search(cursor, - self.user, args)) - # column definitions of the "local" object - _cols = self.pool.get(rec_model)._columns - # if the current field is many2many - if (f_name in _cols) and _cols[f_name]._type=='many2many': - f_val = [(6, 0, [x[f_use] for x in objs])] - elif len(objs): - # otherwise (we are probably in a many2one field), - # take the first element of the search - f_val = objs[0][f_use] - elif len(f_ref): - if f_ref == "null": - f_val = False - else: - f_val = self.id_get(cursor, f_ref) - else: - f_val = _eval_xml(self, field, self.pool, cursor, self.user, - self.idref) - - if model._columns.has_key(f_name): - if isinstance(model._columns[f_name], Integer): - f_val = int(f_val) - res[f_name] = f_val - obj_id = self.pool.get('ir.model.data')._update(cursor, self.user, - rec_model, self.module, res, rec_id or False, - noupdate=self.isnoupdate(data_node), mode=self.mode) - if rec_id: - self.idref[rec_id] = int(obj_id) - return rec_model, obj_id - - def id_get(self, cursor, id_str): - if id_str in self.idref: - return self.idref[id_str] - mod = self.module - if '.' in id_str: - mod, id_str = id_str.split('.') - result = self.pool.get('ir.model.data')._get_id(cursor, self.user, - mod, id_str) - return int(self.pool.get('ir.model.data').read(cursor, self.user, - [result], ['res_id'])[0]['res_id']) - - def parse(self, xmlstr): - doc = xml.dom.minidom.parseString(xmlstr) - elem = doc.documentElement - for node in [i for i in elem.childNodes \ - if (i.nodeType == i.ELEMENT_NODE and i.nodeName=="data")]: - for rec in node.childNodes: - if rec.nodeType == rec.ELEMENT_NODE: - if rec.nodeName in self._tags: - try: - self._tags[rec.nodeName](self.cursor, rec, node) - except: - Logger().notify_channel("init", LOG_INFO, - '\n'+rec.toxml()) - self.cursor.rollback() - raise - return True - - def __init__(self, cursor, module, idref, mode, report=AssertionReport(), - noupdate=False, demo=False): - self.mode = mode - self.module = module - self.cursor = cursor - self.idref = idref - self.pool = pooler.get_pool(cursor.dbname) - self.user = 1 - self.assert_report = report - self.noupdate = noupdate - self.demo = demo - self._tags = { - 'menuitem': self._tag_menuitem, - 'record': self._tag_record, - 'assert': self._tag_assert, - 'report': self._tag_report, - 'wizard': self._tag_wizard, - 'delete': self._tag_delete, - 'ir_set': self._tag_ir_set, - 'function': self._tag_function, - 'workflow': self._tag_workflow, - 'act_window': self._tag_act_window, - } - -def convert_csv_import(cursor, module, fname, csvcontent, idref=None, - mode='init', noupdate=False): - ''' - Import csv file : - quote: " - delimiter: , - encoding: utf-8 - ''' - if idref is None: - idref = {} - model = ('.'.join(fname.split('.')[:-1]).split('-'))[0] - #remove folder path from model - model = os.path.split(model)[1] - - pool = pooler.get_pool(cursor.dbname) - - input_file = StringIO.StringIO(csvcontent) - reader = csv.reader(input_file, quotechar='"', delimiter=',') - fields = reader.next() - - if not (mode == 'init' or 'id' in fields): - return - - user = 1 - datas = [] - for line in reader: - if (not line) or not reduce(lambda x, y: x or y, line) : - continue - datas.append([x.decode('utf8').encode('utf8') for x in line]) - pool.get(model).import_data(cursor, user, fields, datas, mode, - module, noupdate) - -def convert_xml_import_dom(cursor, module, xmlstream, idref=None, mode='init', - noupdate=False, report=None, demo=False): - if idref is None: - idref = {} - if report is None: - report = AssertionReport() - obj = XMLImport(cursor, module, idref, mode, report=report, - noupdate=noupdate, demo=demo) - obj.parse(xmlstream.read()) - del obj - return True - - - - -#$$ -# Notes: - -# - analyser le code _update, source d'amelioration -# - ajouter un handler d'exception -# - faire un cache en debut de traitement : {xml_id: (db_id, model)} -# - prevoir du cache sur les action pour booster la creation des menuitems -# tuto : http://pyxml.sourceforge.net/topics/howto/node14.html from xml import sax @@ -879,10 +119,9 @@ class MenuitemTagHandler: return self else: - res = self.mh.pool.get('ir.model.data')._update( + res = self.mh.pool.get('ir.model.data').import_record( self.mh.cursor, self.mh.user, - 'ir.ui.menu', self.mh.module, self.values, self.xml_id, - noupdate=self.mh.noupdate, mode=self.mh.mode) + 'ir.ui.menu', self.mh.module, self.values, self.xml_id) return None @@ -944,7 +183,6 @@ class RecordTagHandler: self.values[field_name] = answer[0]['id'] elif ref_attr: - # TODO handle correctly the cache on ids self.values[field_name] = self.mh.get_id(ref_attr) elif eval_attr: @@ -1005,15 +243,9 @@ class RecordTagHandler: return self elif name == "record": - # db access: TODO : use the object reference instead of - # the name of the model because _update do a new get to - # obtain the reference - - - res = self.mh.pool.get('ir.model.data')._update( + res = self.mh.pool.get('ir.model.data').import_record( self.mh.cursor, self.mh.user, - self.model._name, self.mh.module, self.values, self.xml_id, - noupdate=self.mh.noupdate, mode=self.mh.mode) + self.model._name, self.mh.module, self.values, self.xml_id) return None else: @@ -1022,12 +254,10 @@ class RecordTagHandler: class TrytondXmlHandler(sax.handler.ContentHandler): - def __init__(self, cursor, pool, mode, module, noupdate): + def __init__(self, cursor, pool, module,): "Register known taghandlers, and manged tags." self.pool = pool - self.mode = mode - self.noupdate = noupdate self.cursor = cursor self.user = 1 self.module = module @@ -1051,13 +281,9 @@ class TrytondXmlHandler(sax.handler.ContentHandler): if '.' in xml_id: module, xml_id = xml_id.split('.') - model_data_id = self.pool.get('ir.model.data')._get_id( + return self.pool.get('ir.model.data').get_id( self.cursor, self.user, module, xml_id) - return int(self.pool.get('ir.model.data').read(self.cursor, self.user, - [model_data_id], ['res_id'])[0]['res_id']) - - def startElement(self, name, attributes): """Rebind the current handler if necessary and call startElement on it""" @@ -1094,13 +320,7 @@ class TrytondXmlHandler(sax.handler.ContentHandler): self.taghandler = self.taghandler.endElement(name) -def convert_xml_import_sax(cursor, module, xmlstream, idref=None, mode='init', - noupdate=False, report=None, demo=False): - if idref is None: - idref = {} - if report is None: - report = AssertionReport() - +def convert_xml_import_sax(cursor, module, xmlstream): parser = sax.make_parser() # Tell the parser we are not interested in XML namespaces @@ -1109,9 +329,8 @@ def convert_xml_import_sax(cursor, module, xmlstream, idref=None, mode='init', handler = TrytondXmlHandler( cursor=cursor, pool=pooler.get_pool(cursor.dbname), - mode=mode, module=module, - noupdate=noupdate,) + ) parser.setContentHandler(handler) source = sax.InputSource() From 5208fd492ce3477d20a70bfb42c1accf1106b7de Mon Sep 17 00:00:00 2001 From: Bertrand Chenal Date: Mon, 7 Jan 2008 23:31:37 +0100 Subject: [PATCH 9/9] Import of data: some speed imporvement and better memory footprint (i hope). --- trytond/ir/model.py | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/trytond/ir/model.py b/trytond/ir/model.py index 7059aa21d2..b8fd0903b4 100644 --- a/trytond/ir/model.py +++ b/trytond/ir/model.py @@ -158,24 +158,37 @@ class ModelData(OSV): def __init__(self, pool): OSV.__init__(self, pool) - # fs2db contain the data of the table ir.model.data. This - # fs2db provide a mapping from the data in the fs to the data - # in the db. This mean that this dict can become very big if a - # lot of files are imported. (improvement store a hash instead of all the values). self.fs2db = None - # imported_records keep tracks of all the records that have - # been imported. - self.imported_records = {} + self.fs2values = None def populate_fs2db(self, cursor, user): - """Fetch all the table in one shot (if necessary).""" + """Fetch all the db_id, model tuple in the table in one shot + (if necessary). This table is kept as a cache for the future + conversions between fs_id abd db_id""" + if not self.fs2db: self.fs2db = {} if cursor.dbname not in self.fs2db: module_data_ids = self.search(cursor, user,[]) self.fs2db[cursor.dbname] = dict([ ((x.fs_id, x.module), - (x.db_id, x.model, x.values)) for x in \ + (x.db_id, x.model)) for x in \ + self.browse(cursor, user, module_data_ids)]) + + def populate_fs2values(self, cursor, user): + """Fetch all the table in one shot (if necessary). When a + record is imported ny import_record, the corresponding item in + fs2values is removed. When post_import is called, the items + remaining in fs2values are the one that were in the model_data + table but not in the fs, so they are removed.""" + # Improvement: store the fiedls names and a hash of the values + # instead of all the values. + if not self.fs2values: self.fs2values = {} + if cursor.dbname not in self.fs2values: + module_data_ids = self.search(cursor, user,[]) + self.fs2values[cursor.dbname] = dict([ + ((x.fs_id, x.module), + x.values) for x in \ self.browse(cursor, user, module_data_ids)]) def get_id(self, cursor, user, module, fs_id): @@ -203,17 +216,14 @@ class ModelData(OSV): module, fs_id = fs_id.split('.') self.populate_fs2db(cursor, user) + self.populate_fs2values(cursor, user) - # keep track of the lits of parsed records. - if cursor.dbname in self.imported_records: - self.imported_records[cursor.dbname].append((fs_id, module)) - else: - self.imported_records[cursor.dbname]= [(fs_id, module)] if (fs_id, module) in self.fs2db[cursor.dbname]: # this record is already in the db: - db_id, db_model, old_values = \ + db_id, db_model = \ self.fs2db[cursor.dbname][(fs_id, module)] + old_values = self.fs2values[cursor.dbname][(fs_id, module)] old_values = eval(old_values) # Check if this record has been modified in the db: @@ -241,8 +251,7 @@ class ModelData(OSV): break if not modified: - # Update the db with the values - # TODO : check if this id exist for this model. + # Update the model_data with the modified values self.pool.get(model).write(cursor, user, db_id, values) self.create(cursor, user, { @@ -254,15 +263,15 @@ class ModelData(OSV): 'date_update': time.strftime('%Y-%m-%d %H:%M:%S'), }) - # Update fs2db: - self.fs2db[cursor.dbname][(fs_id, module)]= (db_id, model, str(values)) else: # We may not overwrite modified data: logger = Logger() logger.notify_channel('init', LOG_INFO, 'Record %s ignored, the corresponding data (%s@%s) ' 'in the db has been modified' % (fs_id,db_id, model)) - + # Remove this record from the value list. This means that + # the corresponding record have been found. + del self.fs2values[cursor.dbname][(fs_id, module)] else: # this record is new db_id = self.pool.get(model).create(cursor, user, values) @@ -281,18 +290,16 @@ class ModelData(OSV): # Test because of a wrong extra call see todo at the end of # load_module_graph in module.py # Globaly this function is a bit dirty. - if not (self.fs2db and self.fs2db.get(cursor.dbname)): + if not (self.fs2values and self.fs2values.get(cursor.dbname)): return True wf_service = LocalService("workflow") data_unlink = [] - for ((fs_id, module),(db_id, model, values)) in \ - self.fs2db[cursor.dbname].items(): - - if module in modules and \ - ((fs_id, module) not in self.imported_records[cursor.dbname]): + for (fs_id, module) in self.fs2values[cursor.dbname]: + if module in modules: + (db_id, model, values) = self.fs2db[cursor.dbname][(fs_id, module)] if model == 'workflow.activity': cursor.execute('SELECT res_type, db_id ' \ @@ -332,8 +339,7 @@ class ModelData(OSV): WHERE (fs_id, module) in (%s) """% ','.join(["('%s','%s')"%x for x in data_unlink])) - del self.fs2db[cursor.dbname] - del self.imported_records[cursor.dbname] + del self.fs2values[cursor.dbname] return True