Improve Binary fields

* Improve memory management of Binary fields
* Support buffer for Binary fields

review84001

CHANGELOG

@@ -1,3 +1,5 @@
+* Improve memory management of Binary fields
+* Support buffer for Binary fields
 * Remove delete on Escape in editable tree
 * Use JSON-RPC
 * Limit size of field when possible

tryton/action/main.py

@@ -9,7 +9,6 @@ from tryton.pyson import PYSONDecoder
 from tryton.exceptions import TrytonServerError
 import gettext
 import tempfile
-import base64
 import os
 import webbrowser
 import tryton.common as common
@@ -62,7 +61,7 @@ class Action(object):
                 name.replace(os.sep, '_').replace(os.altsep or os.sep, '_') \
                         + os.extsep + type)
         with open(fp_name, 'wb') as file_d:
-            file_d.write(base64.decodestring(data))
+            file_d.write(data)
         if email_print:
             mailto(to=email.get('to'), cc=email.get('cc'),
                     subject=email.get('subject'), body=email.get('body'),

tryton/gui/main.py

@@ -30,7 +30,6 @@ from tryton.gui.window.about import About
 from tryton.gui.window.shortcuts import Shortcuts
 from tryton.gui.window.dbrestore import DBRestore
 import re
-import base64
 import tryton.translate as translate
 import tryton.plugins
 import pango
@@ -1310,10 +1309,10 @@ class Main(object):
         url, dbname, passwd, update = dialog.run()
         if dbname:
             with open(filename, 'rb') as file_p:
-                data_b64 = base64.encodestring(file_p.read())
+                data = file_p.read()
             host, port = url.rsplit(':' , 1)
             rpcprogress = common.RPCProgress('db_exec', (host, int(port),
-                'restore', dbname, passwd, data_b64, update))
+                'restore', dbname, passwd, buffer(data), update))
             try:
                 res = rpcprogress.run()
             except TrytonServerError, exception:
@@ -1356,7 +1355,7 @@ class Main(object):
         rpcprogress = common.RPCProgress('db_exec', (host, int(port), 'dump',
             dbname, passwd))
         try:
-            dump_b64 = rpcprogress.run()
+            dump = rpcprogress.run()
         except TrytonServerError, exception:
             if exception.args[0] == "Couldn't dump database with password":
                 common.warning(_("It is not possible to dump a password " \
@@ -1377,7 +1376,6 @@ class Main(object):
             return
 
         self.refresh_ssl()
-        dump = base64.decodestring(dump_b64)
 
         filename = common.file_selection(_('Save As...'),
             action=gtk.FILE_CHOOSER_ACTION_SAVE, preview=False,

tryton/gui/window/view_form/model/field.py

@@ -1,9 +1,11 @@
 #This file is part of Tryton.  The COPYRIGHT file at the top level of
 #this repository contains the full copyright notices and license terms.
+import os
+import tempfile
 import tryton.rpc as rpc
 from tryton.common import DT_FORMAT, DHM_FORMAT, HM_FORMAT, datetime_strftime, \
         domain_inversion, eval_domain, localize_domain, unlocalize_domain, \
-        merge, inverse_leaf, EvalEnvironment
+        merge, inverse_leaf, EvalEnvironment, RPCProgress
 import tryton.common as common
 import time
 import datetime
@@ -790,6 +792,55 @@ class ReferenceField(CharField):
             record.modified_fields.setdefault(self.name)
             record.signal('record-modified')
 
+class BinaryField(CharField):
+
+    def get(self, record, check_load=True, readonly=True, modified=False):
+        result = record.value.get(self.name, False) or False
+        if isinstance(result, basestring):
+            try:
+                with open(result, 'rb') as fp:
+                    result = buffer(fp.read())
+            except IOError:
+                result = self.get_data(record)
+        return result
+
+    def get_client(self, record):
+        return self.get(record)
+
+    def set_client(self, record, value, force_change=False):
+        _, filename = tempfile.mkstemp(prefix='tryton_')
+        with open(filename, 'wb') as fp:
+            fp.write(value)
+        self.set(record, filename)
+        record.modified_fields.setdefault(self.name)
+        record.signal('record-modified')
+        self.sig_changed(record)
+        record.validate(softvalidation=True)
+        record.signal('record-changed')
+
+    def get_size(self, record):
+        result = record.value.get(self.name) or 0
+        if isinstance(result, basestring):
+            result = os.stat(result).st_size
+        return result
+
+    def get_data(self, record):
+        if not isinstance(record.value.get(self.name), basestring):
+            ctx = rpc.CONTEXT.copy()
+            ctx.update(record.context_get())
+            rpcprogress = RPCProgress('execute', ('model', record.model_name,
+                    'read', record.id, [self.name], ctx))
+            try:
+                values = rpcprogress.run()
+            except TrytonServerError, exception:
+                values = common.process_exception(exception)
+                return ''
+            _, filename = tempfile.mkstemp(prefix='tryton_')
+            with open(filename, 'wb') as fp:
+                fp.write(values[self.name])
+            self.set(record, filename)
+        return self.get(record)
+
 TYPES = {
     'char': CharField,
     'sha': CharField,
@@ -807,4 +858,5 @@ TYPES = {
     'datetime': DateTimeField,
     'date': DateField,
     'one2one': O2OField,
+    'binary': BinaryField,
 }

tryton/gui/window/view_form/model/record.py

@@ -75,6 +75,9 @@ class Record(SignalEvent):
             fields.append('_timestamp')
             ctx = rpc.CONTEXT.copy()
             ctx.update(self.context_get())
+            ctx.update(dict(('%s.%s' % (self.model_name, fname), 'size')
+                    for fname, field in self.group.fields.iteritems()
+                    if field.attrs['type'] == 'binary'))
             args = ('model', self.model_name, 'read', ids, fields, ctx)
             try:
                 values = rpc.execute(*args)

tryton/gui/window/view_form/view/form_gtk/binary.py

@@ -1,6 +1,5 @@
 #This file is part of Tryton.  The COPYRIGHT file at the top level of
 #this repository contains the full copyright notices and license terms.
-import base64
 import gtk
 import gettext
 import os
@@ -115,8 +114,7 @@ class Binary(WidgetInterface):
     def sig_new(self, widget=None):
         filename = file_selection(_('Open...'))
         if filename and self.field:
-            self.field.set_client(self.record,
-                    base64.encodestring(open(filename, 'rb').read()))
+            self.field.set_client(self.record, open(filename, 'rb').read())
             if self.filename_field:
                 self.filename_field.set_client(self.record,
                         os.path.basename(filename))
@@ -130,7 +128,7 @@ class Binary(WidgetInterface):
                 os.sep, '_').replace(os.altsep or os.sep, '_')
         file_path = os.path.join(dtemp, filename)
         with open(file_path, 'wb') as fp:
-            fp.write(base64.decodestring(self.field.get(self.record)))
+            fp.write(self.field.get_data(self.record))
         root, type_ = os.path.splitext(filename)
         if type_:
             type_ = type_[1:]
@@ -144,7 +142,7 @@ class Binary(WidgetInterface):
             action=gtk.FILE_CHOOSER_ACTION_SAVE)
         if filename:
             with open(filename,'wb') as fp:
-                fp.write(base64.decodestring(self.field.get(self.record)))
+                fp.write(self.field.get_data(self.record))
 
     def sig_remove(self, widget=None):
         self.field.set_client(self.record, False)
@@ -175,10 +173,10 @@ class Binary(WidgetInterface):
             return False
         if self.wid_text:
             self.wid_text.set_text(self.filename_field.get(record) or '')
-        self.wid_size.set_text(humanize(len(field.get(record) or [])))
+        self.wid_size.set_text(humanize(field.get_size(record) or 0))
         if self.but_open:
-            self.but_open.set_sensitive(bool(field.get(record)))
-        self.but_save_as.set_sensitive(bool(field.get(record)))
+            self.but_open.set_sensitive(bool(field.get_size(record)))
+        self.but_save_as.set_sensitive(bool(field.get_size(record)))
         return True
 
     def set_value(self, record, field):

tryton/gui/window/view_form/view/form_gtk/image.py

@@ -5,7 +5,6 @@ import glib
 import gettext
 import os
 import tempfile
-from base64 import encodestring, decodestring
 from tryton.common import file_selection, Tooltips, file_open
 from tryton.config import PIXMAPS_DIR
 from interface import WidgetInterface
@@ -129,8 +128,7 @@ class Image(WidgetInterface):
         filename = file_selection(_('Open...'), preview=True,
             filters=[filter_image, filter_all])
         if filename:
-            self.field.set_client(self.record,
-                    encodestring(open(filename, 'rb').read()))
+            self.field.set_client(self.record, open(filename, 'rb').read())
             if self.filename_field:
                 self.filename_field.set_client(self.record,
                         os.path.basename(filename))
@@ -144,7 +142,7 @@ class Image(WidgetInterface):
                 os.sep, '_').replace(os.altsep or os.sep, '_')
         file_path = os.path.join(dtemp, filename)
         with open(file_path, 'wb') as fp:
-            fp.write(decodestring(self.field.get(self.record)))
+            fp.write(self.field.get_data(self.record))
         root, type_ = os.path.splitext(filename)
         if type_:
             type_ = type_[1:]
@@ -158,7 +156,7 @@ class Image(WidgetInterface):
             action=gtk.FILE_CHOOSER_ACTION_SAVE)
         if filename:
             with open(filename, 'wb') as fp:
-                fp.write(decodestring(self.field.get(self.record)))
+                fp.write(self.field.get_data(self.record))
 
     def sig_remove(self, widget):
         self.field.set_client(self.record, False)
@@ -177,30 +175,32 @@ class Image(WidgetInterface):
         if info == 0:
             uri = selection.get_text().split('\n')[0]
             if uri:
-                self.field.set_client(self.record,
-                        encodestring(urllib.urlopen(uri).read()))
+                self.field.set_client(self.record, urllib.urlopen(uri).read())
             self.update_img()
         elif info == 1:
             uri = selection.data.split('\r\n')[0]
             if uri:
-                self.field.set_client(self.record,
-                        encodestring(urllib.urlopen(uri).read()))
+                self.field.set_client(self.record, urllib.urlopen(uri).read())
             self.update_img()
         elif info == 2:
             data = selection.get_pixbuf()
             if data:
-                self.field.set_client(self.record,
-                        encodestring(data))
+                self.field.set_client(self.record, data)
                 self.update_img()
 
     def update_img(self):
         value = None
         if self.field:
             value = self.field.get_client(self.record)
+        if isinstance(value, (int, long)):
+            if value > 10**6:
+                value = False
+            else:
+                value = self.field.get_data(self.record)
         if not value:
             data = NOIMAGE
         else:
-            data = decodestring(value)
+            data = value
 
         pixbuf = None
         for ftype in ('jpeg', 'gif', 'png', 'bmp', 'svg'):

tryton/jsonrpc.py

@@ -11,6 +11,7 @@ import gzip
 import StringIO
 import hashlib
 import sys
+import base64
 
 __all__ = ["ResponseError", "Fault", "ProtocolError", "Transport",
     "ServerProxy"]
@@ -37,6 +38,8 @@ def object_hook(dct):
                     dct['hour'], dct['minute'], dct['second'])
         elif dct['__class__'] == 'date':
             return datetime.date(dct['year'], dct['month'], dct['day'])
+        elif dct['__class__'] == 'buffer':
+            return buffer(base64.decodestring(dct['base64']))
         elif dct['__class__'] == 'Decimal':
             return Decimal(dct['decimal'])
     return dct
@@ -60,6 +63,10 @@ class JSONEncoder(json.JSONEncoder):
                     'month': obj.month,
                     'day': obj.day,
                     }
+        elif isinstance(obj, buffer):
+            return {'__class__': 'buffer',
+                'base64': base64.encodestring(obj),
+                }
         elif isinstance(obj, Decimal):
             return {'__class__': 'Decimal',
                 'decimal': str(obj),