tryton-do/tryton
2
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Optimize cache management when importing po file

Browse Source 
issue3356
review1061003
This commit is contained in:
Cédric Krier
2013-09-18 00:13:47 +02:00
parent a3fc52eacc
commit f020d71234
1 changed files with 77 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
trytond/ir/translation.py
+77 -62
View File
@@ -32,6 +32,7 @@ from ..pyson import PYSONEncoder
from ..transaction import Transaction
from ..pool import Pool
from ..cache import Cache
from ..const import RECORD_CACHE_SIZE
__all__ = ['Translation',
'TranslationSetStart', 'TranslationSetSucceed', 'TranslationSet',
@@ -741,13 +742,15 @@ class Translation(ModelSQL, ModelView):
models_data = ModelData.search([
('module', '=', module),
])
fs_id2model_data = {}
fs_id2prop = {}
for model_data in models_data:
fs_id2model_data.setdefault(model_data.model, {})
fs_id2model_data[model_data.model][model_data.fs_id] = model_data
fs_id2prop.setdefault(model_data.model, {})
fs_id2prop[model_data.model][model_data.fs_id] = \
(model_data.db_id, model_data.noupdate)
for extra_model in cls.extra_model_data(model_data):
fs_id2model_data.setdefault(extra_model, {})
fs_id2model_data[extra_model][model_data.fs_id] = model_data
fs_id2prop.setdefault(extra_model, {})
fs_id2prop[extra_model][model_data.fs_id] = \
(model_data.db_id, model_data.noupdate)
translations = set()
to_create = []
@@ -758,11 +761,7 @@ class Translation(ModelSQL, ModelView):
module_translations = cls.search([
('lang', '=', lang),
('module', '=', module),
],
order=[
('type', 'DESC'),
('name', 'DESC'),
])
], order=[])
for translation in module_translations:
if translation.type in ('odt', 'view', 'wizard_button',
'selection', 'error'):
@@ -774,7 +773,8 @@ class Translation(ModelSQL, ModelView):
raise Exception('Unknow translation type: %s'
% translation.type)
key2ids.setdefault(key, []).append(translation.id)
id2translation[translation.id] = translation
if len(module_translations) <= RECORD_CACHE_SIZE:
id2translation[translation.id] = translation
def override_translation(ressource_id, new_translation, fuzzy):
res_id_module, res_id = ressource_id.split('.')
@@ -801,66 +801,81 @@ class Translation(ModelSQL, ModelView):
translation.fuzzy = fuzzy
translation.save()
for entry in pofile:
ttype, name, res_id = entry.msgctxt.split(':')
src = entry.msgid
value = entry.msgstr
fuzzy = 'fuzzy' in entry.flags
noupdate = False
# Make a first loop to retreive translation ids in the right order to
# get better read locality and a full usage of the cache.
translation_ids = []
if len(module_translations) <= RECORD_CACHE_SIZE:
processes = (True,)
else:
processes = (False, True)
for processing in processes:
if processing and len(module_translations) > RECORD_CACHE_SIZE:
id2translation = dict((t.id, t)
for t in Translation.browse(translation_ids))
for entry in pofile:
ttype, name, res_id = entry.msgctxt.split(':')
src = entry.msgid
value = entry.msgstr
fuzzy = 'fuzzy' in entry.flags
noupdate = False
if '.' in res_id:
override_translation(res_id, value, fuzzy)
continue
model = name.split(',')[0]
if model in fs_id2model_data and res_id in fs_id2model_data[model]:
model_data = fs_id2model_data[model][res_id]
res_id = model_data.db_id
noupdate = model_data.noupdate
if res_id:
try:
res_id = int(res_id)
except ValueError:
if '.' in res_id:
override_translation(res_id, value, fuzzy)
continue
if not res_id:
res_id = None
if ttype in ('odt', 'view', 'wizard_button', 'selection', 'error'):
key = (name, res_id, ttype, src)
elif ttype in('field', 'model', 'help'):
key = (name, res_id, ttype)
else:
raise Exception('Unknow translation type: %s' % ttype)
ids = key2ids.get(key, [])
model = name.split(',')[0]
if (model in fs_id2prop
and res_id in fs_id2prop[model]):
res_id, noupdate = fs_id2prop[model][res_id]
with contextlib.nested(Transaction().set_user(0),
Transaction().set_context(module=module)):
if not ids:
to_create.append({
'name': name,
'res_id': res_id,
'lang': lang,
'type': ttype,
'src': src,
'value': value,
'fuzzy': fuzzy,
'module': module,
})
if res_id:
try:
res_id = int(res_id)
except ValueError:
continue
if not res_id:
res_id = None
if ttype in ('odt', 'view', 'wizard_button', 'selection',
'error'):
key = (name, res_id, ttype, src)
elif ttype in('field', 'model', 'help'):
key = (name, res_id, ttype)
else:
translations2 = []
for translation_id in ids:
translation = id2translation[translation_id]
if translation.value != value \
or translation.fuzzy != fuzzy:
translations2.append(translation)
if translations2 and not noupdate:
cls.write(translations2, {
raise Exception('Unknow translation type: %s' % ttype)
ids = key2ids.get(key, [])
if not processing:
translation_ids.extend(ids)
continue
with contextlib.nested(Transaction().set_user(0),
Transaction().set_context(module=module)):
if not ids:
to_create.append({
'name': name,
'res_id': res_id,
'lang': lang,
'type': ttype,
'src': src,
'value': value,
'fuzzy': fuzzy,
'module': module,
'overriding_module': None,
})
translations |= set(cls.browse(ids))
else:
translations2 = []
for translation_id in ids:
translation = id2translation[translation_id]
if translation.value != value \
or translation.fuzzy != fuzzy:
translations2.append(translation)
if translations2 and not noupdate:
cls.write(translations2, {
'value': value,
'fuzzy': fuzzy,
})
translations |= set(cls.browse(ids))
if to_create:
translations |= set(cls.create(to_create))