tryton-do/tryton
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Make cache size configurable

Browse Source 
* Clean private context keyword in RPC
* Add cache section in configuration

issue4320
review10641002
This commit is contained in:
Cédric Krier
2014-11-19 21:35:51 +01:00
parent 66dee3e1d0
commit 360f39f89d
9 changed files with 69 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG
+2
View File
@@ -1,3 +1,5 @@
* Clean private context keyword in RPC
* Add cache section in configuration
* Use dualmethod on ModelStorage.save
* New API for on_change: instance changes
* Add restore_history_before on ModelSQL
doc/topics/configuration.rst
+28
View File
@@ -135,6 +135,34 @@ table for translations.
Default: `en_US`
cache
-----
Defines size of various cache.
model
~~~~~
The number of different model kept in the cache per transaction.
Default: `200`
record
~~~~~~
The number of record loaded kept in the cache of the list.
It can be changed locally using the `_record_cache_size` key in
:attr:`Transaction.context`.
Default: `2000`
field
~~~~~
The number of field to load with an `eager` :attr:`Field.loading`.
Default: `100`
ssl
---
trytond/backend/database.py
+3 -2
View File
@@ -1,6 +1,6 @@
#This file is part of Tryton. The COPYRIGHT file at the top level of
#this repository contains the full copyright notices and license terms.
from trytond.const import MODEL_CACHE_SIZE
from trytond.config import config
DatabaseIntegrityError = None
DatabaseOperationalError = None
@@ -117,7 +117,8 @@ class CursorInterface(object):
context = Transaction().context
keys = tuple(((key, context[key]) for key in sorted(self.cache_keys)
if key in context))
return self.cache.setdefault((user, keys), LRUDict(MODEL_CACHE_SIZE))
return self.cache.setdefault((user, keys),
LRUDict(config.getint('cache', 'model')))
def execute(self, sql, params=None):
'''
trytond/config.py
+4
View File
@@ -51,6 +51,10 @@ class TrytonConfigParser(ConfigParser.RawConfigParser):
self.set('database', 'list', 'True')
self.set('database', 'retry', 5)
self.set('database', 'language', 'en_US')
self.add_section('cache')
self.set('cache', 'model', 200)
self.set('cache', 'record', 2000)
self.set('cache', 'field', 100)
self.add_section('ssl')
self.add_section('email')
self.set('email', 'uri', 'smtp://localhost:25')
trytond/const.py
+1 -4
View File
@@ -1,5 +1,6 @@
#This file is part of Tryton. The COPYRIGHT file at the top level of
#this repository contains the full copyright notices and license terms.
from .config import config
OPERATORS = (
'child_of',
@@ -17,7 +18,3 @@ OPERATORS = (
'<',
'>',
)
MODEL_CACHE_SIZE = 200
RECORD_CACHE_SIZE = 2000
BROWSE_FIELD_TRESHOLD = 100
trytond/ir/translation.py
+6 -4
View File
@@ -27,7 +27,7 @@ from ..pyson import PYSONEncoder
from ..transaction import Transaction
from ..pool import Pool
from ..cache import Cache
from ..const import RECORD_CACHE_SIZE
from ..config import config
__all__ = ['Translation',
'TranslationSetStart', 'TranslationSetSucceed', 'TranslationSet',
@@ -795,7 +795,7 @@ class Translation(ModelSQL, ModelView):
raise ValueError('Unknow translation type: %s' %
translation.type)
key2ids.setdefault(key, []).append(translation.id)
if len(module_translations) <= RECORD_CACHE_SIZE:
if len(module_translations) <= config.getint('cache', 'record'):
id2translation[translation.id] = translation
def override_translation(ressource_id, new_translation):
@@ -829,12 +829,14 @@ class Translation(ModelSQL, ModelView):
# Make a first loop to retreive translation ids in the right order to
# get better read locality and a full usage of the cache.
translation_ids = []
if len(module_translations) <= RECORD_CACHE_SIZE:
if len(module_translations) <= config.getint('cache', 'record'):
processes = (True,)
else:
processes = (False, True)
for processing in processes:
if processing and len(module_translations) > RECORD_CACHE_SIZE:
if (processing
and len(module_translations) > config.getint('cache',
'record')):
id2translation = dict((t.id, t)
for t in cls.browse(translation_ids))
for entry in pofile:
trytond/model/modelsql.py
+7 -4
View File
@@ -15,12 +15,15 @@ from trytond.model import ModelStorage, ModelView
from trytond.model import fields
from trytond import backend
from trytond.tools import reduce_ids, grouped_slice
from trytond.const import OPERATORS, RECORD_CACHE_SIZE
from trytond.const import OPERATORS
from trytond.transaction import Transaction
from trytond.pool import Pool
from trytond.cache import LRUDict
from trytond.exceptions import ConcurrencyException
from trytond.rpc import RPC
from .modelstorage import cache_size
_RE_UNIQUE = re.compile('UNIQUE\s*\((.*)\)', re.I)
_RE_CHECK = re.compile('CHECK\s*\((.*)\)', re.I)
@@ -501,7 +504,7 @@ class ModelSQL(ModelStorage):
cls.__insert_history(new_ids)
records = cls.browse(new_ids)
for sub_records in grouped_slice(records, RECORD_CACHE_SIZE):
for sub_records in grouped_slice(records, cache_size()):
cls._validate(sub_records)
field_names = cls._fields.keys()
@@ -836,7 +839,7 @@ class ModelSQL(ModelStorage):
field.set(cls, fname, *fargs)
cls.__insert_history(all_ids)
for sub_records in grouped_slice(all_records, RECORD_CACHE_SIZE):
for sub_records in grouped_slice(all_records, cache_size()):
cls._validate(sub_records, field_names=all_field_names)
cls.trigger_write(trigger_eligibles)
@@ -1053,7 +1056,7 @@ class ModelSQL(ModelStorage):
rows = cursor.dictfetchmany(cursor.IN_MAX)
cache = cursor.get_cache()
if cls.__name__ not in cache:
cache[cls.__name__] = LRUDict(RECORD_CACHE_SIZE)
cache[cls.__name__] = LRUDict(cache_size())
delete_records = transaction.delete_records.setdefault(cls.__name__,
set())
trytond/model/modelstorage.py
+12 -6
View File
@@ -20,7 +20,8 @@ from trytond.model import Model
from trytond.model import fields
from trytond.tools import reduce_domain, memoize
from trytond.pyson import PYSONEncoder, PYSONDecoder, PYSON
from trytond.const import OPERATORS, RECORD_CACHE_SIZE, BROWSE_FIELD_TRESHOLD
from trytond.const import OPERATORS
from trytond.config import config
from trytond.transaction import Transaction
from trytond.pool import Pool
from trytond.cache import LRUDict, freeze
@@ -32,6 +33,11 @@ from .descriptors import dualmethod
__all__ = ['ModelStorage']
def cache_size():
return Transaction().context.get('_record_cache_size',
config.getint('cache', 'record'))
class ModelStorage(Model):
"""
Define a model with storage capability in Tryton.
@@ -434,7 +440,7 @@ class ModelStorage(Model):
Return a list of instance for the ids
'''
ids = map(int, ids)
local_cache = LRUDict(RECORD_CACHE_SIZE)
local_cache = LRUDict(cache_size())
return [cls(int(x), _ids=ids, _local_cache=local_cache) for x in ids]
@staticmethod
@@ -1151,7 +1157,7 @@ class ModelStorage(Model):
if _local_cache is not None:
self._local_cache = _local_cache
else:
self._local_cache = LRUDict(RECORD_CACHE_SIZE)
self._local_cache = LRUDict(cache_size())
self._local_cache.counter = Transaction().counter
super(ModelStorage, self).__init__(id, **kwargs)
@@ -1160,7 +1166,7 @@ class ModelStorage(Model):
def _cache(self):
cache = self._cursor_cache
if self.__name__ not in cache:
cache[self.__name__] = LRUDict(RECORD_CACHE_SIZE)
cache[self.__name__] = LRUDict(cache_size())
return cache[self.__name__]
def __getattr__(self, name):
@@ -1203,7 +1209,7 @@ class ModelStorage(Model):
to_remove = set(x for x, y in fread_accesses.iteritems()
if not y and x != name)
threshold = BROWSE_FIELD_TRESHOLD
threshold = config.getint('cache', 'field')
def not_cached(item):
fname, field = item
@@ -1284,7 +1290,7 @@ class ModelStorage(Model):
with Transaction().set_context(**ctx):
key = (Model, freeze(ctx))
local_cache = model2cache.setdefault(key,
LRUDict(RECORD_CACHE_SIZE))
LRUDict(cache_size()))
ids = model2ids.setdefault(key, [])
if field._type in ('many2one', 'one2one', 'reference'):
ids.append(value)
trytond/rpc.py
+6 -3
View File
@@ -32,9 +32,12 @@ class RPC(object):
else:
context = args.pop()
timestamp = None
if '_timestamp' in context:
timestamp = context['_timestamp']
del context['_timestamp']
for key in context.keys():
if key == '_timestamp':
timestamp = context[key]
# Remove all private keyword
if key.startswith('_'):
del context[key]
if self.check_access:
context['_check_access'] = True
if self.instantiate is not None: