From 83b1f967a08771743cb310a2c259c6853dd025fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Thu, 4 Aug 2011 16:53:58 +0200 Subject: [PATCH 1/9] Add test for depends review76002 --- trytond/ir/ui/view.py | 4 +-- trytond/res/request.py | 29 ++++++++++---------- trytond/res/user.py | 6 ++--- trytond/test/test.py | 6 +++-- trytond/tests/test_tryton.py | 52 ++++++++++++++++++++++++++++++++++++ trytond/workflow/workflow.py | 4 +-- 6 files changed, 78 insertions(+), 23 deletions(-) diff --git a/trytond/ir/ui/view.py b/trytond/ir/ui/view.py index 73302f7bb5..8280788555 100644 --- a/trytond/ir/ui/view.py +++ b/trytond/ir/ui/view.py @@ -32,11 +32,11 @@ class View(ModelSQL, ModelView): ondelete='CASCADE') field_childs = fields.Char('Children Field', states={ 'invisible': Not(Equal(Eval('type'), 'tree')), - }) + }, depends=['type']) module = fields.Char('Module', readonly=True) domain = fields.Char('Domain', states={ 'invisible': Not(Bool(Eval('inherit'))), - }) + }, depends=['inherit']) def __init__(self): super(View, self).__init__() diff --git a/trytond/res/request.py b/trytond/res/request.py index 33d9d7cda8..235b8cd971 100644 --- a/trytond/res/request.py +++ b/trytond/res/request.py @@ -25,38 +25,39 @@ _READONLY = If(In(Eval('state'), ['waiting', 'closed']), If(Equal(Eval('state'), 'chatting'), Not(Equal(Eval('act_from'), Eval('_user'))), False)) +_DEPENDS = ['state', 'act_from'] class Request(ModelSQL, ModelView): "Request" _name = 'res.request' _description = __doc__ name = fields.Char('Subject', states={ - 'readonly': _READONLY, - }, required=True) + 'readonly': _READONLY, + }, required=True, depends=_DEPENDS) active = fields.Boolean('Active') priority = fields.Selection(_PRIORITIES, 'Priority', states={ - 'readonly': _READONLY, - }, required=True, order_field='priority') + 'readonly': _READONLY, + }, required=True, order_field='priority', depends=_DEPENDS) act_from = fields.Many2One('res.user', 'From', required=True, readonly=True) act_to = fields.Many2One('res.user', 'To', required=True, domain=[('active', '=', True)], states={ 'readonly': _READONLY, - }) + }, depends=_DEPENDS) body = fields.Text('Body', states={ - 'readonly': _READONLY, - }) + 'readonly': _READONLY, + }, depends=_DEPENDS) date_sent = fields.DateTime('Date', readonly=True) trigger_date = fields.DateTime('Trigger Date', states={ - 'readonly': _READONLY, - }) + 'readonly': _READONLY, + }, depends=_DEPENDS) references = fields.One2Many('res.request.reference', 'request', - 'References', states={ - 'readonly': If(Equal(Eval('state'), 'closed'), - True, - Not(Equal(Eval('act_from', 0), Eval('_user', 0)))), - }) + 'References', states={ + 'readonly': If(Equal(Eval('state'), 'closed'), + True, + Not(Equal(Eval('act_from', 0), Eval('_user', 0)))), + }, depends=['state', 'act_from']) number_references = fields.Function(fields.Integer('Number of References', on_change_with=['references']), 'get_number_references') state = fields.Selection(_STATES, 'State', required=True, readonly=True) diff --git a/trytond/res/user.py b/trytond/res/user.py index 082d5d1df3..ef203b32b3 100644 --- a/trytond/res/user.py +++ b/trytond/res/user.py @@ -16,7 +16,7 @@ from trytond.backend import TableHandler from trytond.security import get_connections from trytond.transaction import Transaction from trytond.cache import Cache -from trytond.pyson import Eval, Greater +from trytond.pyson import Eval, Not, Equal from trytond.pool import Pool @@ -31,8 +31,8 @@ class User(ModelSQL, ModelView): signature = fields.Text('Signature') active = fields.Boolean('Active') action = fields.Many2One('ir.action', 'Home Action', states={ - 'required': Greater(Eval('active_id', 0), 0), - }) + 'required': Not(Equal(Eval('login'), 'root')), + }, depends=['login']) menu = fields.Many2One('ir.action', 'Menu Action', domain=[('usage','=','menu')], required=True) groups = fields.Many2Many('res.user-res.group', diff --git a/trytond/test/test.py b/trytond/test/test.py index abca5df215..01c1092d55 100644 --- a/trytond/test/test.py +++ b/trytond/test/test.py @@ -100,7 +100,8 @@ class FloatDigits(ModelSQL): _description = __doc__ digits = fields.Integer('Digits') float = fields.Float(string='Float', help='Test float', - required=False, digits=(16, Eval('digits', 2))) + required=False, digits=(16, Eval('digits', 2)), + depends=['digits']) FloatDigits() @@ -144,7 +145,8 @@ class NumericDigits(ModelSQL): _description = __doc__ digits = fields.Integer('Digits') numeric = fields.Numeric(string='Numeric', help='Test numeric', - required=False, digits=(16, Eval('digits', 2))) + required=False, digits=(16, Eval('digits', 2)), + depends=['digits']) NumericDigits() diff --git a/trytond/tests/test_tryton.py b/trytond/tests/test_tryton.py index 4a4eca4ee7..2bfcd62f1e 100644 --- a/trytond/tests/test_tryton.py +++ b/trytond/tests/test_tryton.py @@ -33,6 +33,7 @@ from trytond.pool import Pool from trytond.backend import Database from trytond.protocols.dispatcher import create from trytond.transaction import Transaction +from trytond.pyson import PYSONEncoder, Eval register_classes() @@ -87,6 +88,22 @@ class ModelViewTestCase(unittest.TestCase): ''' test_view('webdav') + +class FieldDependsTestCase(unittest.TestCase): + ''' + Test Field depends + ''' + + def setUp(self): + install_module('ir') + install_module('res') + install_module('workflow') + install_module('webdav') + + def test0010depends(self): + test_depends() + + def install_module(name): ''' Install module for the tested database @@ -153,6 +170,39 @@ def test_view(module_name): assert field in res['fields'] transaction.cursor.rollback() +def test_depends(): + ''' + Test for missing depends + ''' + class Encoder(PYSONEncoder): + + def __init__(self, *args, **kwargs): + super(Encoder, self).__init__(*args, **kwargs) + self.fields = set() + + def default(self, obj): + if isinstance(obj, Eval): + fname = obj._value + if not fname.startswith('_parent_'): + self.fields.add(fname) + return super(Encoder, self).default(obj) + + with Transaction().start(DB_NAME, USER, CONTEXT) as transaction: + for mname, model in Pool().iterobject(): + for fname, field in model._columns.iteritems(): + encoder = Encoder() + for pyson in (field.domain, field.states): + encoder.encode(pyson) + if hasattr(field, 'digits'): + encoder.encode(field.digits) + encoder.fields.discard(fname) + encoder.fields.discard('context') + encoder.fields.discard('_user') + depends = set(field.depends) + assert encoder.fields <= depends, ( + 'Missing depends %s in "%s"."%s"' % ( + list(encoder.fields - depends), mname, fname)) + def suite(): ''' Return test suite for other modules @@ -174,6 +224,8 @@ def all_suite(): suite_.addTests(test_modelsingleton.suite()) suite_.addTests(unittest.TestLoader( ).loadTestsFromTestCase(ModelViewTestCase)) + suite_.addTests(unittest.TestLoader( + ).loadTestsFromTestCase(FieldDependsTestCase)) import trytond.tests.test_mptt as test_mptt suite_.addTests(test_mptt.suite()) import trytond.tests.test_importdata as test_importdata diff --git a/trytond/workflow/workflow.py b/trytond/workflow/workflow.py index c8969d93f9..21ed2c51ef 100644 --- a/trytond/workflow/workflow.py +++ b/trytond/workflow/workflow.py @@ -72,13 +72,13 @@ class WorkflowActivity(ModelSQL, ModelView): action = fields.Text('Action', states={ 'readonly': Equal(Eval('kind'), 'dummy'), 'required': Equal(Eval('kind'), 'function'), - }) + }, depends=['kind']) flow_start = fields.Boolean('Flow Start') flow_stop = fields.Boolean('Flow Stop') subflow = fields.Many2One('workflow', 'Subflow', states={ 'readonly': Not(Equal(Eval('kind'), 'subflow')), 'required': Equal(Eval('kind'), 'subflow'), - }) + }, depends=['kind']) signal_send = fields.Char('Signal (subflow.*)') out_transitions = fields.One2Many('workflow.transition', 'act_from', 'Outgoing transitions') From 0de081096feb447490fd0657e25b11fbf300b94b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Fri, 5 Aug 2011 01:42:45 +0200 Subject: [PATCH 2/9] Don't make action depend on login login field is not in the preference fields so action can not depend on it because otherwise the preferences dialog will not be valided. issue2125 --- trytond/res/user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trytond/res/user.py b/trytond/res/user.py index ef203b32b3..d0371e2952 100644 --- a/trytond/res/user.py +++ b/trytond/res/user.py @@ -16,7 +16,7 @@ from trytond.backend import TableHandler from trytond.security import get_connections from trytond.transaction import Transaction from trytond.cache import Cache -from trytond.pyson import Eval, Not, Equal +from trytond.pyson import Eval, Bool from trytond.pool import Pool @@ -31,8 +31,8 @@ class User(ModelSQL, ModelView): signature = fields.Text('Signature') active = fields.Boolean('Active') action = fields.Many2One('ir.action', 'Home Action', states={ - 'required': Not(Equal(Eval('login'), 'root')), - }, depends=['login']) + 'required': Bool(Eval('id')), + }, depends=['id']) menu = fields.Many2One('ir.action', 'Menu Action', domain=[('usage','=','menu')], required=True) groups = fields.Many2Many('res.user-res.group', From d5efbc62ad62a1850ae1a123ab4090879f623804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Sat, 6 Aug 2011 14:31:24 +0200 Subject: [PATCH 3/9] Drop NetRPC and activate JSON-RPC by default --- CHANGELOG | 1 + etc/trytond.conf | 11 +- trytond/config.py | 6 +- trytond/protocols/common.py | 103 +++++++++++++- trytond/protocols/jsonrpc.py | 89 +++--------- trytond/protocols/netrpc.py | 139 ------------------- trytond/protocols/webdav.py | 1 - trytond/protocols/xmlrpc.py | 72 +--------- trytond/pysocket.py | 253 ----------------------------------- trytond/server.py | 13 -- 10 files changed, 123 insertions(+), 565 deletions(-) delete mode 100644 trytond/protocols/netrpc.py delete mode 100644 trytond/pysocket.py diff --git a/CHANGELOG b/CHANGELOG index db94a58be5..dfaba16016 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,4 @@ +* Drop NetRPC and activate JSON-RPC by default * Remove ir.action.wizard_size * Rename expand and fill attributes into yexpand and yfill * Add xalign and yalign as field attributes diff --git a/etc/trytond.conf b/etc/trytond.conf index f99ac12a04..f1d439b4ab 100644 --- a/etc/trytond.conf +++ b/etc/trytond.conf @@ -9,18 +9,13 @@ # If empty, the server will listen on all available interfaces #interface = -# Activate the netrpc protocol -#netrpc = True -# Configure the port for the netrpc protocol to listen -#netport = 8070 - # Activate the xml-rpc protocol #xmlrpc = False # Configure the port for the xml-rpc protocol to listen #xmlport = 8069 # Activate the json-rpc protocol -#jsonrpc = False +#jsonrpc = True # Configure the port for the json-rpc protocol to listen #jsonport = 8000 # Configure the path of json-rpc data @@ -57,7 +52,6 @@ #logfile = False # Enable secure connection with SSL -#secure_netrpc = False #secure_xmlrpc = False #secure_jsonrpc = False #secure_webdav = False @@ -75,9 +69,6 @@ # Configure the path to store attachments and sqlite database #data_path = /var/lib/trytond -# Limit the number of threads -#max_thread = 40 - # Allow to run more than one instance of trytond #multi_server = False diff --git a/trytond/config.py b/trytond/config.py index e4c5fe374c..131e2c94de 100644 --- a/trytond/config.py +++ b/trytond/config.py @@ -8,11 +8,9 @@ class ConfigManager(object): self.options = { 'hostname': None, 'interface': '', - 'netrpc': True, - 'netport': 8070, 'xmlrpc': False, 'xmlport': 8069, - 'jsonrpc': False, + 'jsonrpc': True, 'jsonport': 8000, 'jsondata_path': '/var/www/localhost/tryton', 'webdav': False, @@ -31,7 +29,6 @@ class ConfigManager(object): 'debug_mode': False, 'pidfile': None, 'logfile': None, - 'secure_netrpc': False, 'secure_xmlrpc': False, 'secure_jsonrpc': False, 'secure_webdav': False, @@ -44,7 +41,6 @@ class ConfigManager(object): 'smtp_user': False, 'smtp_password': False, 'data_path': '/var/lib/trytond', - 'max_thread': 40, 'multi_server': False, 'session_timeout': 600, 'psyco': False, diff --git a/trytond/protocols/common.py b/trytond/protocols/common.py index e6d0a5cc8b..13092f77dc 100644 --- a/trytond/protocols/common.py +++ b/trytond/protocols/common.py @@ -4,6 +4,13 @@ import errno import os import socket import threading +import select +import sys +import gzip +try: + import cStringIO as StringIO +except ImportError: + import StringIO def endsocket(sock): if os.name != 'nt': @@ -17,7 +24,6 @@ class daemon(threading.Thread): def __init__(self, interface, port, secure, name=None): threading.Thread.__init__(self, name=name) self.secure = secure - self.running = False self.ipv6 = False if socket.has_ipv6: try: @@ -27,11 +33,96 @@ class daemon(threading.Thread): pass def stop(self): - self.running = False + self.server.shutdown() + return def run(self): - self.running = True - while self.running: - self.server.handle_request() - endsocket(self.server.socket) + self.server.serve_forever() return True + +class GZipRequestHandlerMixin: + + if sys.version_info[:2] <= (2, 6): + # Copy from SimpleXMLRPCServer.py with gzip encoding added + def do_POST(self): + """Handles the HTTP POST request. + + Attempts to interpret all HTTP POST requests as XML-RPC calls, + which are forwarded to the server's _dispatch method for handling. + """ + + # Check that the path is legal + if not self.is_rpc_path_valid(): + self.report_404() + return + + try: + # Get arguments by reading body of request. + # We read this in chunks to avoid straining + # socket.read(); around the 10 or 15Mb mark, some platforms + # begin to have problems (bug #792570). + max_chunk_size = 10*1024*1024 + size_remaining = int(self.headers["content-length"]) + L = [] + while size_remaining: + chunk_size = min(size_remaining, max_chunk_size) + L.append(self.rfile.read(chunk_size)) + size_remaining -= len(L[-1]) + data = ''.join(L) + + data = self.decode_request_content(data) + if data is None: + return #response has been sent + + # In previous versions of SimpleXMLRPCServer, _dispatch + # could be overridden in this class, instead of in + # SimpleXMLRPCDispatcher. To maintain backwards compatibility, + # check to see if a subclass implements _dispatch and dispatch + # using that method if present. + response = self.server._marshaled_dispatch( + data, getattr(self, '_dispatch', None) + ) + except Exception: # This should only happen if the module is buggy + # internal error, report as HTTP server error + self.send_response(500) + self.end_headers() + else: + # got a valid XML RPC response + self.send_response(200) + self.send_header("Content-type", "text/xml") + + # Handle gzip encoding + if 'gzip' in self.headers.get('Accept-Encoding', '').split(',') \ + and len(response) > self.encode_threshold: + buffer = StringIO.StringIO() + output = gzip.GzipFile(mode='wb', fileobj=buffer) + output.write(response) + output.close() + buffer.seek(0) + response = buffer.getvalue() + self.send_header('Content-Encoding', 'gzip') + + self.send_header("Content-length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + def decode_request_content(self, data): + #support gzip encoding of request + encoding = self.headers.get("content-encoding", "identity").lower() + if encoding == "identity": + return data + if encoding == "gzip": + f = StringIO.StringIO(data) + gzf = gzip.GzipFile(mode="rb", fileobj=f) + try: + decoded = gzf.read() + except IOError: + self.send_response(400, "error decoding gzip content") + f.close() + gzf.close() + return decoded + else: + self.send_response(501, "encoding %r not supported" % encoding) + self.send_header("Content-length", "0") + self.end_headers() + diff --git a/trytond/protocols/jsonrpc.py b/trytond/protocols/jsonrpc.py index 8933d6b438..ea950b22b7 100644 --- a/trytond/protocols/jsonrpc.py +++ b/trytond/protocols/jsonrpc.py @@ -4,7 +4,7 @@ from trytond.protocols.sslsocket import SSLSocket from trytond.protocols.dispatcher import dispatch from trytond.config import CONFIG from trytond.protocols.datatype import Float -from trytond.protocols.common import daemon +from trytond.protocols.common import daemon, GZipRequestHandlerMixin import SimpleXMLRPCServer import SimpleHTTPServer import SocketServer @@ -12,8 +12,6 @@ import traceback import socket import sys import os -import gzip -import StringIO try: import fcntl except ImportError: @@ -31,6 +29,8 @@ def object_hook(dct): dct['hour'], dct['minute'], dct['second']) elif dct['__class__'] == 'date': return datetime.date(dct['year'], dct['month'], dct['day']) + elif dct['__class__'] == 'Decimal': + return Decimal(dct['decimal']) return dct class JSONEncoder(json.JSONEncoder): @@ -52,7 +52,9 @@ class JSONEncoder(json.JSONEncoder): 'day': obj.day, } elif isinstance(obj, Decimal): - return float(obj) + return {'__class__': 'Decimal', + 'decimal': str(obj), + } return super(JSONEncoder, self).default(obj) @@ -64,7 +66,7 @@ class SimpleJSONRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): reason to instantiate this class directly. """ - def _marshaled_dispatch(self, data, dispatch_method=None): + def _marshaled_dispatch(self, data, dispatch_method=None, path=None): """Dispatches an JSON-RPC method from marshalled (JSON) data. JSON-RPC methods are dispatched from the marshalled (JSON) data @@ -75,7 +77,7 @@ class SimpleJSONRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): existing method through subclassing is the prefered means of changing method dispatch behavior. """ - rawreq = json.loads(data, object_hook=object_hook, parse_float=Float) + rawreq = json.loads(data, object_hook=object_hook) req_id = rawreq.get('id', 0) method = rawreq['method'] @@ -104,7 +106,7 @@ class SimpleJSONRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): traceb = sys.exc_info()[2] pdb.post_mortem(traceb) # report exception back to server - response['error'] = "%s:\n%s" % (sys.exc_value, tb_s) + response['error'] = (str(sys.exc_value), tb_s) return json.dumps(response, cls=JSONEncoder) @@ -126,7 +128,8 @@ class GenericJSONRPCRequestHandler: return res -class SimpleJSONRPCRequestHandler(GenericJSONRPCRequestHandler, +class SimpleJSONRPCRequestHandler(GZipRequestHandlerMixin, + GenericJSONRPCRequestHandler, SimpleXMLRPCServer.SimpleXMLRPCRequestHandler, SimpleHTTPServer.SimpleHTTPRequestHandler): """Simple JSON-RPC request handler class. @@ -134,71 +137,15 @@ class SimpleJSONRPCRequestHandler(GenericJSONRPCRequestHandler, Handles all HTTP POST requests and attempts to decode them as JSON-RPC requests. """ + protocol_version = "HTTP/1.1" rpc_paths = None encode_threshold = 1400 # common MTU - # Copy from SimpleXMLRPCServer.py with gzip encoding added - def do_POST(self): - """Handles the HTTP POST request. - - Attempts to interpret all HTTP POST requests as JSON-RPC calls, - which are forwarded to the server's _dispatch method for handling. - """ - - # Check that the path is legal - if not self.is_rpc_path_valid(): - self.report_404() - return - - try: - # Get arguments by reading body of request. - # We read this in chunks to avoid straining - # socket.read(); around the 10 or 15Mb mark, some platforms - # begin to have problems (bug #792570). - max_chunk_size = 10*1024*1024 - size_remaining = int(self.headers["content-length"]) - L = [] - while size_remaining: - chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) - size_remaining -= len(L[-1]) - data = ''.join(L) - - # In previous versions of SimpleXMLRPCServer, _dispatch - # could be overridden in this class, instead of in - # SimpleXMLRPCDispatcher. To maintain backwards compatibility, - # check to see if a subclass implements _dispatch and dispatch - # using that method if present. - response = self.server._marshaled_dispatch( - data, getattr(self, '_dispatch', None) - ) - except Exception: # This should only happen if the module is buggy - # internal error, report as HTTP server error - self.send_response(500) - self.end_headers() - else: - # got a valid JSON RPC response - self.send_response(200) - self.send_header("Content-type", "application/json-rpc") - - # Handle gzip encoding - if 'gzip' in self.headers.get('Accept-Encoding', '').split(',') \ - and len(response) > self.encode_threshold: - buffer = StringIO.StringIO() - output = gzip.GzipFile(mode='wb', fileobj=buffer) - output.write(response) - output.close() - buffer.seek(0) - response = buffer.getvalue() - self.send_header('Content-Encoding', 'gzip') - - self.send_header("Content-length", str(len(response))) - self.end_headers() - self.wfile.write(response) - - # shut down the connection - self.wfile.flush() - self.connection.shutdown(1) + def send_header(self, keyword, value): + if keyword == 'Content-type' and value == 'text/xml': + value = 'application/json-rpc' + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.send_header(self, + keyword, value) def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. @@ -274,6 +221,7 @@ class SimpleJSONRPCServer(SocketServer.TCPServer, class SimpleThreadedJSONRPCServer(SocketServer.ThreadingMixIn, SimpleJSONRPCServer): timeout = 1 + daemon_threads = True def server_bind(self): self.socket.setsockopt(socket.SOL_SOCKET, @@ -304,7 +252,6 @@ class JSONRPCDaemon(daemon): def __init__(self, interface, port, secure=False): daemon.__init__(self, interface, port, secure, name='JSONRPCDaemon') - if self.secure: handler_class = SecureJSONRPCRequestHandler server_class = SecureThreadedJSONRPCServer diff --git a/trytond/protocols/netrpc.py b/trytond/protocols/netrpc.py deleted file mode 100644 index 238fb208bb..0000000000 --- a/trytond/protocols/netrpc.py +++ /dev/null @@ -1,139 +0,0 @@ -#This file is part of Tryton. The COPYRIGHT file at the top level of -#this repository contains the full copyright notices and license terms. -from trytond.pysocket import PySocket -from trytond.protocols.sslsocket import SSLSocket -from trytond.config import CONFIG -from trytond.protocols.dispatcher import dispatch -from trytond.protocols.common import endsocket -import threading -import select -import traceback -import socket -import sys - - -class NetRPCClientThread(threading.Thread): - def __init__(self, sock, threads, secure): - threading.Thread.__init__(self, name='NetRPCClientThread') - self.sock = sock - self.threads = threads - self.running = False - self.secure = secure - - def run(self): - self.running = True - try: - pysocket = PySocket(self.sock) - except Exception: - self.sock.close() - self.threads.remove(self) - return False - timeout = 0 - while self.running: - try: - rlist, _, _ = select.select([self.sock], [], [], 1) - except select.error: - continue - if not rlist: - timeout += 1 - if timeout > 600: - break - continue - timeout = 0 - try: - msg = pysocket.receive() - except Exception: - pysocket.disconnect() - self.threads.remove(self) - return False - host, port = self.sock.getpeername()[:2] - try: - res = dispatch(host, port, 'NetRPC', *msg) - pysocket.send(res) - except Exception, exception: - tb_s = '' - for line in traceback.format_exception(*sys.exc_info()): - try: - line = line.encode('utf-8', 'ignore') - except Exception: - continue - tb_s += line - for path in sys.path: - tb_s = tb_s.replace(path, '') - if CONFIG['debug_mode']: - import pdb - tback = sys.exc_info()[2] - pdb.post_mortem(tback) - try: - pysocket.send(exception.args, exception=True, - traceback=tb_s) - except Exception: - pysocket.disconnect() - self.threads.remove(self) - return False - pysocket.disconnect() - self.threads.remove(self) - return True - - def stop(self): - self.running = False - - -class NetRPCServerThread(threading.Thread): - def __init__(self, interface, port, secure=False): - threading.Thread.__init__(self, name='NetRPCServerThread') - self.socket = None - if socket.has_ipv6: - try: - socket.getaddrinfo(interface or None, port, socket.AF_INET6) - self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - except Exception: - pass - if self.socket is None: - self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if secure: - self.socket = SSLSocket(self.socket) - self.socket.bind((interface, port)) - self.socket.listen(5) - self.threads = [] - self.secure = secure - self.running = False - - def run(self): - try: - self.running = True - while self.running: - if not int(CONFIG['max_thread']) \ - or len(self.threads) < int(CONFIG['max_thread']): - try: - rlist, _, _ = select.select([self.socket], [], [], 1) - except select.error: - continue - if not rlist: - continue - (clientsocket, _) = self.socket.accept() - c_thread = NetRPCClientThread(clientsocket, self.threads, - self.secure) - self.threads.append(c_thread) - c_thread.start() - except Exception: - try: - self.socket.close() - except Exception: - pass - return False - - def stop(self): - self.running = False - while len(self.threads): - try: - thread = self.threads[0] - thread.stop() - thread.join() - except Exception: - pass - try: - endsocket(self.socket) - except Exception: - return False diff --git a/trytond/protocols/webdav.py b/trytond/protocols/webdav.py index b0be2a9010..96aaa5308d 100644 --- a/trytond/protocols/webdav.py +++ b/trytond/protocols/webdav.py @@ -78,7 +78,6 @@ if not hasattr(WebDAVServer.DAVRequestHandler, '_config'): class BaseThreadedHTTPServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): timeout = 1 - max_children = CONFIG['max_thread'] def server_bind(self): self.socket.setsockopt(socket.SOL_SOCKET, diff --git a/trytond/protocols/xmlrpc.py b/trytond/protocols/xmlrpc.py index ded00e6c3d..137ae45518 100644 --- a/trytond/protocols/xmlrpc.py +++ b/trytond/protocols/xmlrpc.py @@ -4,7 +4,7 @@ from trytond.protocols.sslsocket import SSLSocket from trytond.protocols.dispatcher import dispatch from trytond.config import CONFIG from trytond.protocols.datatype import Float -from trytond.protocols.common import daemon +from trytond.protocols.common import daemon, GZipRequestHandlerMixin from trytond import security import SimpleXMLRPCServer import SocketServer @@ -13,8 +13,6 @@ import traceback import socket import sys import os -import gzip -import StringIO import base64 import datetime from types import DictType @@ -102,8 +100,10 @@ class GenericXMLRPCRequestHandler: security.logout(database_name, user, session) -class SimpleXMLRPCRequestHandler(GenericXMLRPCRequestHandler, +class SimpleXMLRPCRequestHandler(GZipRequestHandlerMixin, + GenericXMLRPCRequestHandler, SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): + protocol_version = "HTTP/1.1" rpc_paths = None encode_threshold = 1400 # common MTU @@ -127,69 +127,6 @@ class SimpleXMLRPCRequestHandler(GenericXMLRPCRequestHandler, self.send_header("WWW-Authenticate", 'Basic realm="Tryton"') return False - # Copy from SimpleXMLRPCServer.py with gzip encoding added - def do_POST(self): - """Handles the HTTP POST request. - - Attempts to interpret all HTTP POST requests as XML-RPC calls, - which are forwarded to the server's _dispatch method for handling. - """ - - # Check that the path is legal - if not self.is_rpc_path_valid(): - self.report_404() - return - - try: - # Get arguments by reading body of request. - # We read this in chunks to avoid straining - # socket.read(); around the 10 or 15Mb mark, some platforms - # begin to have problems (bug #792570). - max_chunk_size = 10*1024*1024 - size_remaining = int(self.headers["content-length"]) - L = [] - while size_remaining: - chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) - size_remaining -= len(L[-1]) - data = ''.join(L) - - # In previous versions of SimpleXMLRPCServer, _dispatch - # could be overridden in this class, instead of in - # SimpleXMLRPCDispatcher. To maintain backwards compatibility, - # check to see if a subclass implements _dispatch and dispatch - # using that method if present. - response = self.server._marshaled_dispatch( - data, getattr(self, '_dispatch', None) - ) - except Exception: # This should only happen if the module is buggy - # internal error, report as HTTP server error - self.send_response(500) - self.end_headers() - else: - # got a valid XML RPC response - self.send_response(200) - self.send_header("Content-type", "text/xml") - - # Handle gzip encoding - if 'gzip' in self.headers.get('Accept-Encoding', '').split(',') \ - and len(response) > self.encode_threshold: - buffer = StringIO.StringIO() - output = gzip.GzipFile(mode='wb', fileobj=buffer) - output.write(response) - output.close() - buffer.seek(0) - response = buffer.getvalue() - self.send_header('Content-Encoding', 'gzip') - - self.send_header("Content-length", str(len(response))) - self.end_headers() - self.wfile.write(response) - - # shut down the connection - self.wfile.flush() - self.connection.shutdown(1) - class SecureXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): @@ -202,6 +139,7 @@ class SecureXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer.SimpleXMLRPCServer): timeout = 1 + daemon_threads = True def server_bind(self): self.socket.setsockopt(socket.SOL_SOCKET, diff --git a/trytond/pysocket.py b/trytond/pysocket.py deleted file mode 100644 index abbcf7daae..0000000000 --- a/trytond/pysocket.py +++ /dev/null @@ -1,253 +0,0 @@ -#This file is part of Tryton. The COPYRIGHT file at the top level of -#this repository contains the full copyright notices and license terms. -import socket -# can't use/fall-back pickle due to different interface :-( -import cPickle -try: - import cStringIO as StringIO -except ImportError: - import StringIO -try: - import ssl -except ImportError: - ssl = None -import gzip -try: - import hashlib -except ImportError: - hashlib = None - import sha -import os - -DNS_CACHE = {} -MAX_SIZE = 999999999 -MAX_LENGHT = len(str(MAX_SIZE)) -CONNECT_TIMEOUT = 5 -TIMEOUT = 3600 -GZIP_THRESHOLD = 1400 # common MTU - -_ALLOWED_MODULES = {'datetime': ['datetime', 'date'], 'decimal': ['Decimal']} - -def checkfunction(module, klass): - if module in _ALLOWED_MODULES and klass in _ALLOWED_MODULES[module]: - mod = __import__(module, {}, {}, ['__all__']) - _class = getattr(mod, klass) - return _class - raise ValueError('Not supported: %s/%s' % (module, klass)) - - -class PySocket: - - def __init__(self, sock=None, fingerprints=None, ca_certs=None): - self.sock = sock - self.host = None - self.hostname = None - self.port = None - self.ssl = False - self.ssl_sock = None - self.connected = False - self.buffer = '' - self.fingerprints = fingerprints - if ca_certs and os.path.isfile(ca_certs): - self.ca_certs = ca_certs - else: - self.ca_certs = None - - def connect(self, host, port=False): - if not port: - buf = host.split('//')[1] - host, port = buf.rsplit(':', 1) - hostname = host - if host in DNS_CACHE: - host = DNS_CACHE[host] - self.sock = None - if socket.has_ipv6: - try: - socket.getaddrinfo(host, int(port), socket.AF_INET6) - self.sock = socket.socket(socket.AF_INET6, - socket.SOCK_STREAM) - self.sock.settimeout(CONNECT_TIMEOUT) - self.sock.connect((host, int(port))) - except socket.error: - self.sock = None - if self.sock is None: - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.settimeout(CONNECT_TIMEOUT) - self.sock.connect((host, int(port))) - DNS_CACHE[hostname], port = self.sock.getpeername()[:2] - try: - sock = None - if socket.has_ipv6: - try: - socket.getaddrinfo(host, int(port), socket.AF_INET6) - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - sock.settimeout(CONNECT_TIMEOUT) - sock.connect((host, int(port))) - except socket.error: - sock = None - if sock is None: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(CONNECT_TIMEOUT) - sock.connect((host, int(port))) - if ssl: - ssl_sock = ssl.wrap_socket(sock) - self.ssl = True - elif hasattr(socket, 'ssl'): - ssl_sock = socket.ssl(sock) - self.ssl = True - except socket.error: - pass - self.sock.settimeout(TIMEOUT) - fingerprint = None - if self.ssl: - if ssl: - self.ssl_sock = ssl.wrap_socket(self.sock, - ca_certs=self.ca_certs, cert_reqs=(self.ca_certs and - ssl.CERT_REQUIRED or ssl.CERT_NONE)) - elif hasattr(socket, 'ssl'): - self.ssl_sock = socket.ssl(self.sock, certfile=self.ca_certs) - try: - peercert = self.ssl_sock.getpeercert(True) - except socket.error: - peercert = None - def format_hash(value): - return reduce(lambda x, y: x + y[1].upper() + - ((y[0] % 2 and y[0] + 1 < len(value)) and ':' or ''), - enumerate(value), '') - if peercert and hashlib: - fingerprint = format_hash(hashlib.sha1(peercert).hexdigest()) - elif peercert: - fingerprint = format_hash(sha1.new(peercert).hexdigest()) - self.host = host - self.hostname = hostname - self.port = port - self.connected = True - self.buffer = '' - if self.fingerprints is not None: - key = (self.hostname, str(self.port)) - if key in self.fingerprints: - if self.fingerprints[key] != fingerprint: - self.disconnect() - raise Exception('BadFingerprint') - elif fingerprint: - self.fingerprints[key] = fingerprint - - def disconnect(self): - try: - sock = self.sock - if self.ssl: - sock = self.ssl_sock - try: - shutdown_value = 2 - if hasattr(socket, 'SHUT_RDWR'): - shutdown_value = socket.SHUT_RDWR - if hasattr(sock, 'sock_shutdown'): - sock.sock_shutdown(shutdown_value) - else: - sock.shutdown(shutdown_value) - except socket.error: - pass - sock.close() - except socket.error: - pass - self.sock = None - self.ssl = False - self.ssl_sock = None - self.connected = False - self.buffer = '' - - def reconnect(self): - if self.hostname and self.port: - self.disconnect() - self.connect(self.hostname, self.port) - - def send(self, msg, exception=False, traceback=None): - msg = cPickle.dumps([msg, traceback], protocol=2) - gzip_p = False - if len(msg) > GZIP_THRESHOLD: - buffer = StringIO.StringIO() - output = gzip.GzipFile(mode='wb', fileobj=buffer) - output.write(msg) - output.close() - buffer.seek(0) - msg = buffer.getvalue() - gzip_p = True - size = len(msg) - msg = str(size) + ' ' + (exception and "1" or "0") \ - + (gzip_p and "1" or "0") + msg - size = len(msg) - - if self.ssl: - totalsent = 0 - while totalsent < size: - sent = self.ssl_sock.write(msg[totalsent:]) - if sent == 0: - raise RuntimeError("socket connection broken") - totalsent = totalsent + sent - else: - self.sock.sendall(msg) - - def receive(self): - buf = self.buffer - L = [] - size_remaining = MAX_LENGHT - while size_remaining: - chunk_size = min(size_remaining, 4096) - if self.ssl: - chunk = self.ssl_sock.read(chunk_size) - else: - chunk = self.sock.recv(chunk_size) - if chunk == '': - raise RuntimeError("socket connection broken") - L.append(chunk) - size_remaining -= len(chunk) - if ' ' in chunk: - break - if size_remaining < 0: - raise RuntimeError("socket connection broken") - buf += ''.join(L) - size, msg = buf.split(' ', 1) - size = int(size) - if size > MAX_SIZE: - raise RuntimeError("socket connection broken") - while len(msg) < 2: - chunk_size = min(size + 2, 4096) - if self.ssl: - msg += self.ssl_sock.read(chunk_size) - else: - msg += self.sock.recv(chunk_size) - if msg == '': - raise RuntimeError("socket connection broken") - exception = msg[0] != "0" - gzip_p = msg[1] != "0" - L = [msg[2:]] - size_remaining = size - len(L[0]) - while size_remaining: - chunk_size = min(size_remaining, 4096) - if self.ssl: - chunk = self.ssl_sock.read(chunk_size) - else: - chunk = self.sock.recv(chunk_size) - if chunk == '': - raise RuntimeError("socket connection broken") - L.append(chunk) - size_remaining -= len(chunk) - msg = ''.join(L) - if len(msg) > size: - self.buffer = msg[size:] - msg = msg[:size] - else: - self.buffer = '' - msgio = StringIO.StringIO(msg) - if gzip_p: - output = gzip.GzipFile(mode='r', fileobj=msgio) - msgio = StringIO.StringIO(output.read(-1)) - output.close() - unpickler = cPickle.Unpickler(msgio) - # cPickle mechanism to import instances (pickle differs here) - unpickler.find_global = checkfunction - res = unpickler.load() - if exception: - raise Exception(*(list(res[0]) + [res[1]])) - else: - return res[0] diff --git a/trytond/server.py b/trytond/server.py index ccb5f96a32..79a9c1dd35 100644 --- a/trytond/server.py +++ b/trytond/server.py @@ -170,14 +170,6 @@ class TrytonServer(object): (CONFIG['secure_jsonrpc'] and ' Secure' or '', CONFIG['jsonport'])) - if CONFIG['netrpc']: - from trytond.protocols.netrpc import NetRPCServerThread - netrpcd = NetRPCServerThread(CONFIG['interface'], CONFIG['netport'], - CONFIG['secure_netrpc']) - self.logger.info("starting NetRPC%s protocol, port %d" % \ - (CONFIG['secure_netrpc'] and ' Secure' or '', - CONFIG['netport'])) - if CONFIG['webdav']: from trytond.protocols.webdav import WebDAVServerThread webdavd = WebDAVServerThread(CONFIG['interface'], @@ -191,9 +183,6 @@ class TrytonServer(object): if signum == signal.SIGUSR1: Pool.start() return - if CONFIG['netrpc']: - netrpcd.stop() - netrpcd.join() if CONFIG['xmlrpc']: xmlrpcd.stop() xmlrpcd.join() @@ -221,8 +210,6 @@ class TrytonServer(object): signal.signal(signal.SIGUSR1, handler) self.logger.info('waiting for connections...') - if CONFIG['netrpc']: - netrpcd.start() if CONFIG['xmlrpc']: xmlrpcd.start() if CONFIG['jsonrpc']: From d26164af74cdeb0cbaa70750032c7ed13fc00c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Sat, 6 Aug 2011 18:25:24 +0200 Subject: [PATCH 4/9] Use unoconv and all Opendocument format in report issue2031 review72001 --- CHANGELOG | 2 + doc/topics/install.rst | 2 +- doc/topics/reports/index.rst | 3 + etc/trytond.conf | 3 + setup.py | 2 +- trytond/config.py | 1 + trytond/ir/action.py | 85 ++++++++++++++++++++++++-- trytond/ir/action.xml | 4 +- trytond/report/report.py | 114 +++++++++++++++++------------------ 9 files changed, 148 insertions(+), 68 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index dfaba16016..7e33dc5ca0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +* Use unoconv for report format conversion +* Allow to use any Opendocument as report template * Drop NetRPC and activate JSON-RPC by default * Remove ir.action.wizard_size * Rename expand and fill attributes into yexpand and yfill diff --git a/doc/topics/install.rst b/doc/topics/install.rst index ed496ac66a..dfa3832437 100644 --- a/doc/topics/install.rst +++ b/doc/topics/install.rst @@ -17,7 +17,7 @@ Prerequisites * Optional: pywebdav 0.9.3 or later (http://code.google.com/p/pywebdav/) * Optional: pydot (http://code.google.com/p/pydot/) * Optional: pytz (http://pytz.sourceforge.net/) - * Optional: openoffice-python (http://pypi.python.org/pypi/openoffice-python/) + * Optional: unoconv http://dag.wieers.com/home-made/unoconv/) * Optional: sphinx (http://sphinx.pocoo.org/) Install Tryton diff --git a/doc/topics/reports/index.rst b/doc/topics/reports/index.rst index 96a70b11ad..7adeab0f2a 100644 --- a/doc/topics/reports/index.rst +++ b/doc/topics/reports/index.rst @@ -74,6 +74,8 @@ When defining an `ir.action.report` the following attributes are available: example, my_module/my_style.odt. If no style is to be used this field will be blank. + * ``template_extension``: The template format. + Report Usage ============ @@ -211,6 +213,7 @@ Then you must activate the new invoice report that exists in your new module: account.invoice my_module/invoice.odt module_name/header_A4.odt + odt Passing custom data to a report diff --git a/etc/trytond.conf b/etc/trytond.conf index f1d439b4ab..85dc4cf004 100644 --- a/etc/trytond.conf +++ b/etc/trytond.conf @@ -87,3 +87,6 @@ # Enable cron # cron = True + +# unoconv connection +#unoconv = pipe,name=trytond;urp;StarOffice.ComponentContext diff --git a/setup.py b/setup.py index e21a377c6b..004c7b1dc8 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ setup(name=PACKAGE, 'PostgreSQL': ['psycopg2 >= 2.0'], 'MySQL': ['MySQL-python'], 'WebDAV': ['PyWebDAV >= 0.9.3'], - 'PDF': ['openoffice-python'], + 'unoconv': ['unoconv'], 'SSL': ['pyOpenSSL'], 'graphviz': ['pydot'], 'timezone': ['pytz'], diff --git a/trytond/config.py b/trytond/config.py index 131e2c94de..04068f0a0c 100644 --- a/trytond/config.py +++ b/trytond/config.py @@ -49,6 +49,7 @@ class ConfigManager(object): 'init': {}, 'update': {}, 'cron': True, + 'unoconv': 'pipe,name=trytond;urp;StarOffice.ComponentContext', } self.configfile = None diff --git a/trytond/ir/action.py b/trytond/ir/action.py index 8be92988f6..9198b1be24 100644 --- a/trytond/ir/action.py +++ b/trytond/ir/action.py @@ -226,10 +226,76 @@ class ActionReport(ModelSQL, ModelView): style_content = fields.Function(fields.Binary('Style'), 'get_style_content') direct_print = fields.Boolean('Direct Print') - extension = fields.Selection( - [('odt', 'ODT Document'), - ('pdf', 'PDF Document'),], - string='Extension', required=True) + template_extension = fields.Selection([ + ('odt', 'OpenDocument Text'), + ('odp', 'OpenDocument Presentation'), + ('ods', 'OpenDocument Spreadsheet'), + ('odg', 'OpenDocument Graphics'), + ], string='Template Extension', required=True) + extension = fields.Selection([ + ('', ''), + ('bib', 'BibTex'), + ('bmp', 'Windows Bitmap'), + ('csv', 'Text CSV'), + ('dbf', 'dBase'), + ('dif', 'Data Interchange Format'), + ('doc', 'Microsoft Word 97/2000/XP'), + ('doc6', 'Microsoft Word 6.0'), + ('doc95', 'Microsoft Word 95'), + ('docbook', 'DocBook'), + ('emf', 'Enhanced Metafile'), + ('eps', 'Encapsulated PostScript'), + ('gif', 'Graphics Interchange Format'), + ('html', 'HTML Document'), + ('jpg', 'Joint Photographic Experts Group'), + ('met', 'OS/2 Metafile'), + ('ooxml', 'Microsoft Office Open XML'), + ('pbm', 'Portable Bitmap'), + ('pct', 'Mac Pict'), + ('pdb', 'AportisDoc (Palm)'), + ('pdf', 'Portable Document Format'), + ('pgm', 'Portable Graymap'), + ('png', 'Portable Network Graphic'), + ('ppm', 'Portable Pixelmap'), + ('ppt', 'Microsoft PowerPoint 97/2000/XP'), + ('psw', 'Pocket Word'), + ('pwp', 'PlaceWare'), + ('pxl', 'Pocket Excel'), + ('ras', 'Sun Raster Image'), + ('rtf', 'Rich Text Format'), + ('latex', 'LaTeX 2e'), + ('sda', 'StarDraw 5.0 (OpenOffice.org Impress)'), + ('sdc', 'StarCalc 5.0'), + ('sdc4', 'StarCalc 4.0'), + ('sdc3', 'StarCalc 3.0'), + ('sdd', 'StarImpress 5.0'), + ('sdd3', 'StarDraw 3.0 (OpenOffice.org Impress)'), + ('sdd4', 'StarImpress 4.0'), + ('sdw', 'StarWriter 5.0'), + ('sdw4', 'StarWriter 4.0'), + ('sdw3', 'StarWriter 3.0'), + ('slk', 'SYLK'), + ('svg', 'Scalable Vector Graphics'), + ('svm', 'StarView Metafile'), + ('swf', 'Macromedia Flash (SWF)'), + ('sxc', 'OpenOffice.org 1.0 Spreadsheet'), + ('sxi', 'OpenOffice.org 1.0 Presentation'), + ('sxd', 'OpenOffice.org 1.0 Drawing'), + ('sxd3', 'StarDraw 3.0'), + ('sxd5', 'StarDraw 5.0'), + ('sxw', 'Open Office.org 1.0 Text Document'), + ('text', 'Text Encoded'), + ('tiff', 'Tagged Image File Format'), + ('txt', 'Plain Text'), + ('wmf', 'Windows Metafile'), + ('xhtml', 'XHTML Document'), + ('xls', 'Microsoft Excel 97/2000/XP'), + ('xls5', 'Microsoft Excel 5.0'), + ('xls95', 'Microsoft Excel 95'), + ('xpm', 'X PixMap'), + ], + string='Extension', help='Leave empty for the same as template, ' + 'see unoconv documentation for compatible format') module = fields.Char('Module', readonly=True, select=1) email = fields.Char('Email') pyson_email = fields.Function(fields.Char('PySON Email'), 'get_pyson') @@ -269,6 +335,12 @@ class ActionReport(ModelSQL, ModelView): TableHandler.dropTable(cursor, 'ir.action.report.outputformat', 'ir_action_report_outputformat') + # Migrate from 2.0 remove required on extension + table.not_null_action('extension', action='remove') + cursor.execute('UPDATE "' + self._table + '" ' + 'SET extension = %s ' + 'WHERE extension = %s', ('', 'odt')) + def default_type(self): return 'ir.action.report' @@ -278,9 +350,12 @@ class ActionReport(ModelSQL, ModelView): def default_direct_print(self): return False - def default_extension(self): + def default_template_extension(self): return 'odt' + def default_extension(self): + return '' + def default_module(self): return Transaction().context.get('module') or '' diff --git a/trytond/ir/action.xml b/trytond/ir/action.xml index 08a28fbad6..9a5318655b 100644 --- a/trytond/ir/action.xml +++ b/trytond/ir/action.xml @@ -92,9 +92,11 @@ this repository contains the full copyright notices and license terms. -->