Stream json and gzip response

Closes #14409

trytond/CHANGELOG

@@ -1,3 +1,4 @@
+* Stream json and gzip response
 * Allow users to subscribe to notification cron jobs
 * Add user notification
 * Add support for materializing ModelSQL based on a table query

trytond/trytond/protocols/jsonrpc.py

@@ -2,7 +2,6 @@
 # this repository contains the full copyright notices and license terms.
 import base64
 import datetime
-import gzip
 import json
 from decimal import Decimal
 from types import MappingProxyType
@@ -15,7 +14,7 @@ from werkzeug.wrappers import Response
 from trytond.exceptions import (
     ConcurrencyException, LoginException, MissingDependenciesException,
     RateLimitException, TrytonException, UserWarning)
-from trytond.protocols.wrappers import Request
+from trytond.protocols.wrappers import GzipStream, Request
 from trytond.tools import cached_property
 
 
@@ -154,6 +153,20 @@ class JSONRequest(Request):
             raise BadRequest("Unable to get RPC params") from e
 
 
+encoder = JSONEncoder(separators=(',', ':'))
+
+
+def dumps(obj, limit=1400):
+    chunks = []
+    total = 0
+    for chunk in encoder.iterencode(obj):
+        total += len(chunk)
+        if total > limit:
+            raise OverflowError()
+        chunks.append(chunk)
+    return ''.join(chunks)
+
+
 class JSONProtocol:
     content_type = 'json'
 
@@ -196,10 +209,13 @@ class JSONProtocol:
                 return InternalServerError(data)
             response = data
         headers = {}
-        data = json.dumps(
-            response, cls=JSONEncoder, separators=(',', ':'))
-        if len(data) >= 1400 and 'gzip' in request.accept_encodings:
-            data = gzip.compress(data.encode('utf-8'), compresslevel=1)
+
+        try:
+            payload = dumps(response)
+        except OverflowError:
+            payload = encoder.iterencode(response)
+            if 'gzip' in request.accept_encodings:
+                payload = GzipStream(payload, compresslevel=1)
             headers['Content-Encoding'] = 'gzip'
         return Response(
-            data, content_type='application/json', headers=headers)
+            payload, content_type='application/json', headers=headers)

trytond/trytond/protocols/wrappers.py

@@ -4,6 +4,7 @@ import base64
 import gzip
 import logging
 import time
+import zlib
 from functools import wraps
 
 try:
@@ -309,3 +310,21 @@ def user_application(name, json=True):
             return response
         return wrapper
     return decorator
+
+
+class GzipStream:
+    def __init__(self, data, compresslevel=6):
+        if isinstance(data, str):
+            data = [data]
+        self.iterator = data
+        self.compressor = zlib.compressobj(level=compresslevel, wbits=31)
+
+    def __iter__(self):
+        for chunk in self.iterator:
+            data = chunk.encode('utf-8')
+            compressed = self.compressor.compress(data)
+            if compressed:
+                yield compressed
+        tail = self.compressor.flush()
+        if tail:
+            yield tail

trytond/trytond/protocols/xmlrpc.py

@@ -1,7 +1,6 @@
 # This file is part of Tryton.  The COPYRIGHT file at the top level of
 # this repository contains the full copyright notices and license terms.
 import datetime
-import gzip
 import logging
 import xmlrpc.client as client
 from decimal import Decimal
@@ -17,7 +16,7 @@ from trytond.exceptions import (
     ConcurrencyException, LoginException, MissingDependenciesException,
     RateLimitException, TrytonException, UserWarning)
 from trytond.model.fields.dict import ImmutableDict
-from trytond.protocols.wrappers import Request
+from trytond.protocols.wrappers import GzipStream, Request
 from trytond.tools import cached_property
 
 logger = logging.getLogger(__name__)
@@ -180,7 +179,7 @@ class XMLProtocol:
             data = client.dumps(
                 data, methodresponse=True, allow_none=True)
             if len(data) >= 1400 and 'gzip' in request.accept_encodings:
-                data = gzip.compress(data.encode('utf-8'), compresslevel=1)
+                data = GzipStream(data.encode('utf-8'), compresslevel=1)
                 headers['Content-Encoding'] = 'gzip'
             return Response(
                 data, content_type='text/xml', headers=headers)