diff --git a/trytond/cache.py b/trytond/cache.py new file mode 100644 index 0000000000..fb70b81678 --- /dev/null +++ b/trytond/cache.py @@ -0,0 +1,168 @@ +#This file is part of Tryton. The COPYRIGHT file at the top level of +#this repository contains the full copyright notices and license terms. +import inspect +import time +from threading import Lock +from trytond.transaction import Transaction +from trytond.config import CONFIG +from trytond.backend import Database + + +class Cache(object): + """ + Use it as a decorator of the function you plan to cache + Timeout: 0 = no timeout, otherwise in seconds + """ + _cache_instance = [] + _resets = {} + _resets_lock = Lock() + + def __init__(self, name, timeout=3600, max_len=1024): + self.timeout = timeout + self.max_len = max_len + self._cache = {} + self._cache_instance.append(self) + self._name = name + self._timestamp = None + self._lock = Lock() + + def __call__(self, function): + arg_names = inspect.getargspec(function)[0] + + def reset(): + dbname = Transaction().cursor.dbname + Cache.reset(dbname, self._name) + self._lock.acquire() + try: + self._cache[dbname] = {} + finally: + self._lock.release() + return True + + def call(model, *args, **kwargs): + result = None + find = False + cursor = Transaction().cursor + # Update named arguments with positional argument values + kwargs_origin = kwargs.copy() + kwargs.update(dict(zip(arg_names, args))) + kwargs = kwargs.items() + kwargs.sort() + + self._lock.acquire() + try: + self._cache.setdefault(cursor.dbname, {}) + finally: + self._lock.release() + + lower = None + self._lock.acquire() + try: + if len(self._cache[cursor.dbname]) > self.max_len: + mintime = time.time() - self.timeout + for key in self._cache[cursor.dbname].keys(): + last_time = self._cache[cursor.dbname][key][1] + if mintime > last_time: + del self._cache[cursor.dbname][key] + else: + if not lower or lower[1] > last_time: + lower = (key, last_time) + if (len(self._cache[cursor.dbname]) > self.max_len + and lower): + del self._cache[cursor.dbname][lower[0]] + finally: + self._lock.release() + + # Work out key as a tuple + key = (id(model), Transaction().user, repr(Transaction().context), + repr(kwargs)) + + # Check cache and return cached value if possible + self._lock.acquire() + try: + if key in self._cache[cursor.dbname]: + (value, last_time) = self._cache[cursor.dbname][key] + mintime = time.time() - self.timeout + if self.timeout <= 0 or mintime <= last_time: + self._cache[cursor.dbname][key] = (value, + time.time()) + result = value + find = True + finally: + self._lock.release() + + if not find: + # Work out new value, cache it and return it + result = function(model, *args, **kwargs_origin) + + self._lock.acquire() + try: + self._cache[cursor.dbname][key] = (result, time.time()) + finally: + self._lock.release() + return result + + call.__doc__ = function.__doc__ + call.reset = reset + return call + + @staticmethod + def clean(dbname): + if not CONFIG['multi_server']: + return + database = Database(dbname).connect() + cursor = database.cursor() + try: + cursor.execute('SELECT "timestamp", "name" FROM ir_cache') + timestamps = {} + for timestamp, name in cursor.fetchall(): + timestamps[name] = timestamp + finally: + cursor.commit() + cursor.close() + for obj in Cache._cache_instance: + if obj._name in timestamps: + if not obj._timestamp or timestamps[obj._name] > obj._timestamp: + obj._timestamp = timestamps[obj._name] + obj._lock.acquire() + try: + obj._cache[dbname] = {} + finally: + obj._lock.release() + + @staticmethod + def reset(dbname, name): + if not CONFIG['multi_server']: + return + Cache._resets_lock.acquire() + try: + Cache._resets.setdefault(dbname, set()) + Cache._resets[dbname].add(name) + finally: + Cache._resets_lock.release() + return + + @staticmethod + def resets(dbname): + if not CONFIG['multi_server']: + return + database = Database(dbname).connect() + cursor = database.cursor() + Cache._resets_lock.acquire() + Cache._resets.setdefault(dbname, set()) + try: + for name in Cache._resets[dbname]: + cursor.execute('SELECT name FROM ir_cache WHERE name = %s', + (name,)) + if cursor.fetchone(): + cursor.execute('UPDATE ir_cache SET "timestamp" = %s '\ + 'WHERE name = %s', (datetime.datetime.now(), name)) + else: + cursor.execute('INSERT INTO ir_cache ' \ + '("timestamp", "name") ' \ + 'VALUES (%s, %s)', (datetime.datetime.now(), name)) + Cache._resets[dbname].clear() + finally: + cursor.commit() + cursor.close() + Cache._resets_lock.release() diff --git a/trytond/tools/convert.py b/trytond/convert.py similarity index 100% rename from trytond/tools/convert.py rename to trytond/convert.py diff --git a/trytond/ir/action.py b/trytond/ir/action.py index 44c2e3a6be..abbe8bb469 100644 --- a/trytond/ir/action.py +++ b/trytond/ir/action.py @@ -4,10 +4,11 @@ from __future__ import with_statement import base64 import os from trytond.model import ModelView, ModelSQL, ModelStorage, fields -from trytond.tools import file_open, Cache, safe_eval +from trytond.tools import file_open, safe_eval from trytond.backend import TableHandler from trytond.pyson import PYSONEncoder, CONTEXT, PYSON from trytond.transaction import Transaction +from trytond.cache import Cache class Action(ModelSQL, ModelView): diff --git a/trytond/ir/default.py b/trytond/ir/default.py index 358ba80441..1a0fe0ec3a 100644 --- a/trytond/ir/default.py +++ b/trytond/ir/default.py @@ -4,9 +4,10 @@ from decimal import Decimal import datetime import time from trytond.model import ModelView, ModelSQL, fields -from trytond.tools import Cache, safe_eval +from trytond.tools import safe_eval from trytond.pyson import Eval from trytond.transaction import Transaction +from trytond.cache import Cache class Default(ModelSQL, ModelView): diff --git a/trytond/ir/model.py b/trytond/ir/model.py index 64f5e9271d..35ef47d47a 100644 --- a/trytond/ir/model.py +++ b/trytond/ir/model.py @@ -4,10 +4,10 @@ import datetime import base64 import re from trytond.model import ModelView, ModelSQL, fields -from trytond.tools import Cache from trytond.report import Report from trytond.wizard import Wizard from trytond.transaction import Transaction +from trytond.cache import Cache IDENTIFIER = re.compile(r'^[a-zA-z_][a-zA-Z0-9_]*$') diff --git a/trytond/ir/rule.py b/trytond/ir/rule.py index c79023bf1d..9bf70e6e3f 100644 --- a/trytond/ir/rule.py +++ b/trytond/ir/rule.py @@ -4,9 +4,10 @@ from __future__ import with_statement import contextlib import time from trytond.model import ModelView, ModelSQL, fields -from trytond.tools import Cache, safe_eval +from trytond.tools import safe_eval from trytond.pyson import Eval, Get from trytond.transaction import Transaction +from trytond.cache import Cache class RuleGroup(ModelSQL, ModelView): diff --git a/trytond/ir/trigger.py b/trytond/ir/trigger.py index 8f280e727c..af7a84d0d7 100644 --- a/trytond/ir/trigger.py +++ b/trytond/ir/trigger.py @@ -4,10 +4,11 @@ import datetime import time from trytond.model import ModelView, ModelSQL, fields from trytond.pyson import Eval, Or -from trytond.tools import Cache, safe_eval +from trytond.tools import safe_eval from trytond.backend import TableHandler from trytond.tools import reduce_ids from trytond.transaction import Transaction +from trytond.cache import Cache class Trigger(ModelSQL, ModelView): diff --git a/trytond/model/cacheable.py b/trytond/model/cacheable.py index 3f79d0e872..a1407fd87b 100644 --- a/trytond/model/cacheable.py +++ b/trytond/model/cacheable.py @@ -2,8 +2,8 @@ #this repository contains the full copyright notices and license terms. from threading import Lock import time -from trytond.tools import Cache from trytond.transaction import Transaction +from trytond.cache import Cache class Cacheable(object): diff --git a/trytond/model/modelview.py b/trytond/model/modelview.py index 9ee0847b5d..a4292bf484 100644 --- a/trytond/model/modelview.py +++ b/trytond/model/modelview.py @@ -8,10 +8,10 @@ except ImportError: import md5 import copy from trytond.model import Model -from trytond.tools import Cache, safe_eval -from trytond.modules import create_graph, get_module_list +from trytond.tools import safe_eval from trytond.pyson import PYSONEncoder, CONTEXT from trytond.transaction import Transaction +from trytond.cache import Cache def _find(tree, element): if element.tag == 'xpath': @@ -82,6 +82,7 @@ class ModelView(Model): ModelView.__modules_list = None def _get_modules_list(self): + from trytond.modules import create_graph, get_module_list if ModelView.__modules_list: return ModelView.__modules_list graph = create_graph(get_module_list())[0] diff --git a/trytond/modules/__init__.py b/trytond/modules/__init__.py index 34d28ec9d7..b9e5a08d18 100644 --- a/trytond/modules/__init__.py +++ b/trytond/modules/__init__.py @@ -9,10 +9,11 @@ import traceback import logging import contextlib from trytond.backend import Database -from trytond.tools import Cache import trytond.tools as tools from trytond.config import CONFIG from trytond.transaction import Transaction +from trytond.cache import Cache +import trytond.convert as convert OPJ = os.path.join MODULES_PATH = os.path.abspath(os.path.dirname(__file__)) @@ -234,7 +235,7 @@ def load_module_graph(graph, pool, lang=None): models_to_update_history.add(model._name) #Instanciate a new parser for the package: - tryton_parser = tools.TrytondXmlHandler(pool=pool, module=module) + tryton_parser = convert.TrytondXmlHandler(pool=pool, module=module) for filename in package.datas.get('xml', []): filename = filename.replace('/', os.sep) @@ -302,7 +303,7 @@ def load_module_graph(graph, pool, lang=None): # Vacuum : while modules_todo: (module, to_delete) = modules_todo.pop() - tools.post_import(pool, module, to_delete) + convert.post_import(pool, module, to_delete) cursor.commit() diff --git a/trytond/protocols/dispatcher.py b/trytond/protocols/dispatcher.py index 1605cbdafd..2c9e19214d 100644 --- a/trytond/protocols/dispatcher.py +++ b/trytond/protocols/dispatcher.py @@ -2,14 +2,6 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from __future__ import with_statement -from trytond.pool import Pool -from trytond import security -from trytond.tools import Cache -from trytond.backend import Database -from trytond.config import CONFIG -from trytond.version import VERSION -from trytond.monitor import monitor -from trytond.transaction import Transaction import traceback import logging import time @@ -21,6 +13,14 @@ except ImportError: import sha import base64 import pydoc +from trytond.pool import Pool +from trytond import security +from trytond.backend import Database +from trytond.config import CONFIG +from trytond.version import VERSION +from trytond.monitor import monitor +from trytond.transaction import Transaction +from trytond.cache import Cache def dispatch(host, port, protocol, database_name, user, session, object_type, object_name, method, *args, **kargs): diff --git a/trytond/protocols/webdav.py b/trytond/protocols/webdav.py index aceed17412..a0799bd488 100644 --- a/trytond/protocols/webdav.py +++ b/trytond/protocols/webdav.py @@ -22,10 +22,11 @@ from trytond.protocols.sslsocket import SSLSocket from trytond.config import CONFIG from trytond.security import login from trytond.version import PACKAGE, VERSION, WEBSITE -from trytond.tools.misc import Cache, LocalDict +from trytond.tools.misc import LocalDict from trytond.backend import Database from trytond.pool import Pool from trytond.transaction import Transaction +from trytond.cache import Cache _TRYTON_RELOAD = False domimpl = xml.dom.minidom.getDOMImplementation() diff --git a/trytond/res/user.py b/trytond/res/user.py index f35cf41c80..d95f98d231 100644 --- a/trytond/res/user.py +++ b/trytond/res/user.py @@ -3,12 +3,6 @@ "User" from __future__ import with_statement import copy -from trytond.model import ModelView, ModelSQL, fields -from trytond.wizard import Wizard -from trytond.tools import Cache, safe_eval -from trytond.backend import TableHandler -from trytond.security import get_connections -from trytond.transaction import Transaction import string import random try: @@ -16,6 +10,13 @@ try: except ImportError: hashlib = None import sha +from trytond.model import ModelView, ModelSQL, fields +from trytond.wizard import Wizard +from trytond.tools import safe_eval +from trytond.backend import TableHandler +from trytond.security import get_connections +from trytond.transaction import Transaction +from trytond.cache import Cache from trytond.pyson import Eval, Greater diff --git a/trytond/tools/__init__.py b/trytond/tools/__init__.py index c710b3d7c1..21c264dc2e 100644 --- a/trytond/tools/__init__.py +++ b/trytond/tools/__init__.py @@ -1,5 +1,4 @@ #This file is part of Tryton. The COPYRIGHT file at the top level of #this repository contains the full copyright notices and license terms. from misc import * -from convert import * from datetime_strftime import * diff --git a/trytond/tools/misc.py b/trytond/tools/misc.py index 562732539c..194948caff 100644 --- a/trytond/tools/misc.py +++ b/trytond/tools/misc.py @@ -4,22 +4,18 @@ """ Miscelleanous tools used by tryton """ -import os, time, sys -import inspect -from trytond.config import CONFIG +import os, sys import subprocess import zipfile -from trytond.backend import Database -from threading import Lock, local +from threading import local import smtplib try: import cStringIO as StringIO except ImportError: import StringIO import dis -import datetime from decimal import Decimal -from trytond.transaction import Transaction +from trytond.config import CONFIG def find_in_path(name): if os.name == "nt": @@ -172,166 +168,6 @@ def find_language_context(args, kargs=None): res = arg.get('language', 'en_US') return kargs.get('context', {}).get('language', res) - -class Cache(object): - """ - Use it as a decorator of the function you plan to cache - Timeout: 0 = no timeout, otherwise in seconds - """ - _cache_instance = [] - _resets = {} - _resets_lock = Lock() - - def __init__(self, name, timeout=3600, max_len=1024): - self.timeout = timeout - self.max_len = max_len - self._cache = {} - self._cache_instance.append(self) - self._name = name - self._timestamp = None - self._lock = Lock() - - def __call__(self, function): - arg_names = inspect.getargspec(function)[0] - - def reset(): - dbname = Transaction().cursor.dbname - Cache.reset(dbname, self._name) - self._lock.acquire() - try: - self._cache[dbname] = {} - finally: - self._lock.release() - return True - - def call(model, *args, **kwargs): - result = None - find = False - cursor = Transaction().cursor - # Update named arguments with positional argument values - kwargs_origin = kwargs.copy() - kwargs.update(dict(zip(arg_names, args))) - kwargs = kwargs.items() - kwargs.sort() - - self._lock.acquire() - try: - self._cache.setdefault(cursor.dbname, {}) - finally: - self._lock.release() - - lower = None - self._lock.acquire() - try: - if len(self._cache[cursor.dbname]) > self.max_len: - mintime = time.time() - self.timeout - for key in self._cache[cursor.dbname].keys(): - last_time = self._cache[cursor.dbname][key][1] - if mintime > last_time: - del self._cache[cursor.dbname][key] - else: - if not lower or lower[1] > last_time: - lower = (key, last_time) - if (len(self._cache[cursor.dbname]) > self.max_len - and lower): - del self._cache[cursor.dbname][lower[0]] - finally: - self._lock.release() - - # Work out key as a tuple - key = (id(model), Transaction().user, repr(Transaction().context), - repr(kwargs)) - - # Check cache and return cached value if possible - self._lock.acquire() - try: - if key in self._cache[cursor.dbname]: - (value, last_time) = self._cache[cursor.dbname][key] - mintime = time.time() - self.timeout - if self.timeout <= 0 or mintime <= last_time: - self._cache[cursor.dbname][key] = (value, - time.time()) - result = value - find = True - finally: - self._lock.release() - - if not find: - # Work out new value, cache it and return it - result = function(model, *args, **kwargs_origin) - - self._lock.acquire() - try: - self._cache[cursor.dbname][key] = (result, time.time()) - finally: - self._lock.release() - return result - - call.__doc__ = function.__doc__ - call.reset = reset - return call - - @staticmethod - def clean(dbname): - if not CONFIG['multi_server']: - return - database = Database(dbname).connect() - cursor = database.cursor() - try: - cursor.execute('SELECT "timestamp", "name" FROM ir_cache') - timestamps = {} - for timestamp, name in cursor.fetchall(): - timestamps[name] = timestamp - finally: - cursor.commit() - cursor.close() - for obj in Cache._cache_instance: - if obj._name in timestamps: - if not obj._timestamp or timestamps[obj._name] > obj._timestamp: - obj._timestamp = timestamps[obj._name] - obj._lock.acquire() - try: - obj._cache[dbname] = {} - finally: - obj._lock.release() - - @staticmethod - def reset(dbname, name): - if not CONFIG['multi_server']: - return - Cache._resets_lock.acquire() - try: - Cache._resets.setdefault(dbname, set()) - Cache._resets[dbname].add(name) - finally: - Cache._resets_lock.release() - return - - @staticmethod - def resets(dbname): - if not CONFIG['multi_server']: - return - database = Database(dbname).connect() - cursor = database.cursor() - Cache._resets_lock.acquire() - Cache._resets.setdefault(dbname, set()) - try: - for name in Cache._resets[dbname]: - cursor.execute('SELECT name FROM ir_cache WHERE name = %s', - (name,)) - if cursor.fetchone(): - cursor.execute('UPDATE ir_cache SET "timestamp" = %s '\ - 'WHERE name = %s', (datetime.datetime.now(), name)) - else: - cursor.execute('INSERT INTO ir_cache ' \ - '("timestamp", "name") ' \ - 'VALUES (%s, %s)', (datetime.datetime.now(), name)) - Cache._resets[dbname].clear() - finally: - cursor.commit() - cursor.close() - Cache._resets_lock.release() - def memoize(maxsize): """ Decorator to 'memoize' a function - caching its results with a