Add row factory to cursor

Closes #14797
This commit is contained in:
Cédric Krier committed 2026-06-04 23:41:21 +02:00
1 parent 8d8a5ffa14
commit 5f56aa0769
15 files changed
+142 -96

No files matched your search

+4 -4
View File
@@ -19,8 +19,7 @@ from trytond.model.exceptions import AccessError
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If, PYSONEncoder
from trytond.tools import (
cursor_dict, is_full_text, lstrip_wildcard, sqlite_apply_types)
from trytond.tools import is_full_text, lstrip_wildcard, sqlite_apply_types
from trytond.transaction import Transaction
from trytond.wizard import Button, StateAction, StateView, Wizard
@@ -884,7 +883,7 @@ class Tax(sequence_ordered(), ModelSQL, ModelView, DeactivableMixin):
Move = pool.get('account.move')
MoveLine = pool.get('account.move.line')
TaxLine = pool.get('account.tax.line')
cursor = Transaction().connection.cursor()
connection = Transaction().connection
move = Move.__table__()
move_line = MoveLine.__table__()
@@ -943,8 +942,9 @@ class Tax(sequence_ordered(), ModelSQL, ModelView, DeactivableMixin):
if backend.name == 'sqlite':
sqlite_apply_types(query, [None] + ['NUMERIC'] * len(names))
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*query)
for row in cursor_dict(cursor):
for row in cursor:
for name in names:
result[name][row['tax']] = row[name] or 0
return result
+9 -9
View File
@@ -21,7 +21,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval, If
from trytond.rpc import RPC
from trytond.tools import cursor_dict, sortable_values, sqlite_apply_types
from trytond.tools import sortable_values, sqlite_apply_types
from trytond.transaction import Transaction
from trytond.wizard import StateAction, Wizard
@@ -182,7 +182,7 @@ class Group(ModelSQL, ModelView, ChatMixin):
def get_payment_aggregated(cls, groups, names):
pool = Pool()
Payment = pool.get('account.payment')
cursor = Transaction().connection.cursor()
connection = Transaction().connection
payment = Payment.__table__()
@@ -207,16 +207,16 @@ class Group(ModelSQL, ModelView, ChatMixin):
if backend.name == 'sqlite':
sqlite_apply_types(
query, [None, None, 'NUMERIC', 'NUMERIC', None])
cursor = connection.cursor(row_factory=backend.namedtuple_row)
cursor.execute(*query)
for row in cursor_dict(cursor):
group = cls(row['group_id'])
for row in cursor:
group = cls(row.group_id)
result['payment_count'][group.id] = row['payment_count']
result['payment_complete'][group.id] = \
not row['payment_not_complete']
result['payment_count'][group.id] = row.payment_count
result['payment_complete'][group.id] = not row.payment_not_complete
amount = row['payment_amount']
succeeded = row['payment_amount_succeeded']
amount = row.payment_amount
succeeded = row.payment_amount_succeeded
if amount is not None and backend.name == 'sqlite':
amount = group.company.currency.round(amount)
+7 -6
View File
@@ -12,7 +12,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Eval
from trytond.rpc import RPC
from trytond.tools import cursor_dict, sqlite_apply_types
from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
@@ -125,7 +125,7 @@ class Journal(DeactivableMixin, ModelSQL, ModelView):
Statement = pool.get('account.statement')
statement = Statement.__table__()
transaction = Transaction()
cursor = transaction.connection.cursor()
connection = transaction.connection
id2currency = {j.id: j.currency for j in journals}
@@ -156,12 +156,13 @@ class Journal(DeactivableMixin, ModelSQL, ModelView):
if 'last_date' in names:
types.append('DATE')
sqlite_apply_types(query, types)
cursor = connection.cursor(row_factory=backend.namedtuple_row)
cursor.execute(*query)
for row in cursor_dict(cursor):
journal = row['journal']
for row in cursor:
journal = row.journal
if 'last_amount' in names:
result['last_amount'][journal] = (
id2currency[journal].round(row['last_amount']))
id2currency[journal].round(row.last_amount))
if 'last_date' in names:
result['last_date'][journal] = row['last_date']
result['last_date'][journal] = row.last_date
return result
@@ -12,12 +12,13 @@ from sql.aggregate import Sum
from sql.conditionals import Case
from sql.operators import Exists
from trytond import backend
from trytond.i18n import gettext
from trytond.model import (
DeactivableMixin, MatchMixin, ModelSQL, ModelView, fields)
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, Id, If
from trytond.tools import cursor_dict, decistmt
from trytond.tools import decistmt
from trytond.tools.decimal_ import DecimalNull
from trytond.transaction import Transaction
@@ -541,7 +542,7 @@ class ExciseDeclarationProduct(ModelSQL, ModelView):
Move = pool.get('stock.move')
transaction = Transaction()
context = transaction.context
cursor = transaction.connection.cursor()
connection = transaction.connection
move = Move.__table__()
location = Location.__table__()
@@ -578,8 +579,9 @@ class ExciseDeclarationProduct(ModelSQL, ModelView):
declaration_ids = [s.id for s in declarations]
query.where = where & fields.SQL_OPERATORS['in'](
move.product, declaration_ids)
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*query)
for values in cursor_dict(cursor):
for values in cursor:
declaration = id2declaration[values['id']]
for name in names:
if qty := values[name]:
@@ -614,7 +616,7 @@ class ExciseDeclarationProduct(ModelSQL, ModelView):
Move = pool.get('stock.move')
transaction = Transaction()
context = transaction.context
cursor = transaction.connection.cursor()
connection = transaction.connection
move = Move.__table__()
location = Location.__table__()
@@ -651,8 +653,9 @@ class ExciseDeclarationProduct(ModelSQL, ModelView):
declaration_ids = [s.id for s in declarations]
query.where = where & fields.SQL_OPERATORS['in'](
move.product, declaration_ids)
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*query)
for values in cursor_dict(cursor):
for values in cursor:
declaration = id2declaration[values['id']]
for name in names:
if qty := values[name]:
+2
View File
@@ -1,3 +1,5 @@
* Remove cursor_dict
* Add row factory to cursor
* Add filename extension to Binary field
* Add filters attribute for binary and image widgets
* Move last_user and last_modification to ResourceAccessMixin
+15 -1
View File
@@ -20,6 +20,19 @@ The backend manages the database connection and schema manipulation.
The maximum number of parameters supported by the backend.
.. attribute:: dict_row
A row factory for cursor returning rows as :py:class:`dictionary <dict>`.
.. attribute:: namedtuple_row
A row factory for cursor returning rows as
:py:func:`~collections.namedtuple`.
.. attribute:: scalar_row
A row factory for cursor returning the first column as a scalar value.
Database
========
@@ -33,7 +46,8 @@ Database
.. method:: Database.get_connection([autocommit[, readonly]])
Retrieve a connection object as defined by :pep:`249#connection`.
Retrieve a connection object as defined by :pep:`249#connection`, bearing in
mind that the ``cursor()`` method accepts a row_factory keyword parameter.
If autocommit is set, the connection is committed after each statement.
If readonly is set, the connection is read only.
+5 -1
View File
@@ -13,7 +13,8 @@ import trytond.config as config
__all__ = [
'name', 'Database', 'TableHandler',
'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError',
'DatabaseTimeoutError', 'MAX_QUERY_PARAMS']
'DatabaseTimeoutError', 'MAX_QUERY_PARAMS',
'dict_row', 'namedtuple_row', 'scalar_row']
name = urllib.parse.urlparse(config.get('database', 'uri', default='')).scheme
@@ -38,3 +39,6 @@ DatabaseOperationalError = _module.DatabaseOperationalError
DatabaseTimeoutError = _module.DatabaseTimeoutError
TableHandler = _module.TableHandler
MAX_QUERY_PARAMS = _module.MAX_QUERY_PARAMS
dict_row = _module.dict_row
namedtuple_row = _module.namedtuple_row
scalar_row = _module.scalar_row
@@ -3,7 +3,8 @@
from .database import (
Database, DatabaseDataError, DatabaseIntegrityError,
DatabaseOperationalError, DatabaseTimeoutError)
DatabaseOperationalError, DatabaseTimeoutError, dict_row, namedtuple_row,
scalar_row)
from .table import TableHandler
MAX_QUERY_PARAMS = 50_000 # rounded down from 65_535
@@ -14,6 +15,9 @@ __all__ = [
DatabaseIntegrityError,
DatabaseOperationalError,
DatabaseTimeoutError,
dict_row,
namedtuple_row,
scalar_row,
MAX_QUERY_PARAMS,
TableHandler,
]
@@ -16,7 +16,7 @@ from psycopg import DataError as DatabaseDataError
from psycopg import IntegrityError as DatabaseIntegrityError
from psycopg import IsolationLevel
from psycopg import OperationalError as DatabaseOperationalError
from psycopg import connect
from psycopg import connect, rows
from psycopg.errors import QueryCanceled as DatabaseTimeoutError
from psycopg.sql import SQL, Identifier
from psycopg_pool import ConnectionPool
@@ -35,7 +35,8 @@ from .table import index_method
__all__ = [
'Database',
'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError',
'DatabaseTimeoutError']
'DatabaseTimeoutError',
'dict_row', 'namedtuple_row', 'scalar_row']
logger = logging.getLogger(__name__)
@@ -63,6 +64,11 @@ class LoggingCursor(Cursor):
return super().execute(query, params, prepare=prepare, binary=binary)
dict_row = rows.dict_row
namedtuple_row = rows.namedtuple_row
scalar_row = rows.scalar_row
class ForSkipLocked(For):
def __str__(self):
assert not self.nowait, "Can not use both NO WAIT and SKIP LOCKED"
+5 -1
View File
@@ -6,7 +6,8 @@ import sys
from .database import (
Database, DatabaseDataError, DatabaseIntegrityError,
DatabaseOperationalError, DatabaseTimeoutError)
DatabaseOperationalError, DatabaseTimeoutError, dict_row, namedtuple_row,
scalar_row)
from .table import TableHandler
MAX_QUERY_PARAMS = 200 # estimation from the SQLITE_MAX_EXPR_DEPTH=1_000
@@ -20,6 +21,9 @@ __all__ = [
DatabaseIntegrityError,
DatabaseOperationalError,
DatabaseTimeoutError,
dict_row,
namedtuple_row,
scalar_row,
MAX_QUERY_PARAMS,
TableHandler,
]
+22 -3
View File
@@ -10,6 +10,7 @@ import threading
import time
import urllib.parse
import warnings
from collections import namedtuple
from decimal import Decimal
from sqlite3 import DatabaseError
from sqlite3 import IntegrityError as DatabaseIntegrityError
@@ -32,7 +33,8 @@ from trytond.transaction import Transaction
__all__ = [
'Database',
'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError',
'DatabaseTimeoutError']
'DatabaseTimeoutError',
'dict_row', 'namedtuple_row', 'scalar_row']
logger = logging.getLogger(__name__)
_default_name = config.get('database', 'default_name', default=':memory:')
@@ -455,8 +457,25 @@ class SQLiteCursor(sqlite.Cursor):
class SQLiteConnection(sqlite.Connection):
def cursor(self):
return super().cursor(SQLiteCursor)
def cursor(self, *, row_factory=None):
cursor = super().cursor(SQLiteCursor)
cursor.row_factory = row_factory
return cursor
def dict_row(cursor, row):
fields = [column[0] for column in cursor.description]
return {key: value for key, value in zip(fields, row)}
def namedtuple_row(cursor, row):
fields = [column[0] for column in cursor.description]
cls = namedtuple("Row", fields)
return cls._make(row)
def scalar_row(cursor, row):
return row[0]
class Database(DatabaseInterface):
+7 -4
View File
@@ -12,6 +12,7 @@ from sql.conditionals import Case
from sql.operators import Equal
import trytond.config as config
from trytond import backend
from trytond.cache import Cache
from trytond.i18n import gettext
from trytond.model import (
@@ -22,7 +23,7 @@ from trytond.pool import Pool
from trytond.pyson import Bool, Eval, PYSONDecoder
from trytond.report import Report
from trytond.rpc import RPC
from trytond.tools import cursor_dict, is_instance_method
from trytond.tools import is_instance_method
from trytond.tools.string_ import StringMatcher
from trytond.transaction import Transaction, without_check_access
from trytond.wizard import (
@@ -352,11 +353,13 @@ class ModelField(
pool = Pool()
Model = pool.get('ir.model')
cursor = Transaction().connection.cursor()
connection = Transaction().connection
cursor_select = connection.cursor(row_factory=backend.dict_row)
cursor = connection.cursor()
ir_model_field = cls.__table__()
cursor.execute(*ir_model_field
cursor_select.execute(*ir_model_field
.select(
ir_model_field.id.as_('id'),
ir_model_field.name.as_('name'),
@@ -367,7 +370,7 @@ class ModelField(
ir_model_field.help.as_('help'),
ir_model_field.access.as_('access'),
where=ir_model_field.model == model.__name__))
model_fields = {f['name']: f for f in cursor_dict(cursor)}
model_fields = {f['name']: f for f in cursor_select}
for field_name, field in model._fields.items():
if hasattr(field, 'get_target'):
+35 -41
View File
@@ -17,13 +17,14 @@ from sql.functions import CharLength, Position, Substring
from sql.operators import Concat
import trytond.config as config
from trytond import backend
from trytond.cache import Cache
from trytond.exceptions import UserError
from trytond.i18n import gettext
from trytond.model import Index, ModelSQL, ModelView, fields
from trytond.pool import Pool
from trytond.pyson import Eval, PYSONEncoder
from trytond.tools import cursor_dict, file_open
from trytond.tools import file_open
from trytond.tools.string_ import LazyString, StringPartitioned
from trytond.transaction import (
Transaction, inactive_records, without_check_access)
@@ -185,7 +186,7 @@ class Translation(
@classmethod
def register_fields(cls, model, module_name):
cursor = Transaction().connection.cursor()
connection = Transaction().connection
ir_translation = cls.__table__()
# Prefetch field translations
@@ -195,6 +196,7 @@ class Translation(
selection=defaultdict(dict))
if model._fields:
names = ['%s,%s' % (model.__name__, f) for f in model._fields]
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*ir_translation.select(ir_translation.id,
ir_translation.name, ir_translation.src,
ir_translation.type,
@@ -202,13 +204,14 @@ class Translation(
& ir_translation.type.in_(
('field', 'help', 'selection'))
& ir_translation.name.in_(names))))
for trans in cursor_dict(cursor):
for trans in cursor:
sources = translations[trans['type']][trans['name']]
sources[trans['src']] = trans
columns = [ir_translation.name, ir_translation.lang,
ir_translation.type, ir_translation.src, ir_translation.value,
ir_translation.module, ir_translation.fuzzy, ir_translation.res_id]
cursor = connection.cursor()
def insert(field, type, name, string):
inserted = False
@@ -241,16 +244,19 @@ class Translation(
@classmethod
def register_wizard(cls, wizard, module_name):
cursor = Transaction().connection.cursor()
connection = Transaction().connection
ir_translation = cls.__table__()
# Prefetch button translations
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*ir_translation.select(
ir_translation.id, ir_translation.name, ir_translation.src,
where=((ir_translation.lang == INTERNAL_LANG)
& (ir_translation.type == 'wizard_button')
& (ir_translation.name.like(wizard.__name__ + ',%')))))
trans_buttons = {t['name']: t for t in cursor_dict(cursor)}
trans_buttons = {t['name']: t for t in cursor}
cursor = connection.cursor()
def update_insert_button(state_name, button):
if not button.string:
@@ -1178,9 +1184,11 @@ class TranslationSet(Wizard):
else:
return
cursor = Transaction().connection.cursor()
connection = Transaction().connection
translation = Translation.__table__()
report_strings = defaultdict(set)
cursor = connection.cursor()
cursor_select = connection.cursor(row_factory=backend.dict_row)
for report in reports:
content = None
if report.report:
@@ -1193,7 +1201,7 @@ class TranslationSet(Wizard):
if not content:
continue
cursor.execute(*translation.select(
cursor_select.execute(*translation.select(
translation.id, translation.name,
translation.src, translation.src_plural,
where=(translation.lang == INTERNAL_LANG)
@@ -1202,7 +1210,7 @@ class TranslationSet(Wizard):
& (translation.module == module)))
trans_reports = {
(t['src'], t['src_plural']): t
for t in cursor_dict(cursor)}
for t in cursor_select}
strings = report_strings[report.report_name, report.module]
func_name = 'extract_report_%s' % report.template_extension
@@ -1271,16 +1279,18 @@ class TranslationSet(Wizard):
else:
return
cursor = Transaction().connection.cursor()
connection = Transaction().connection
translation = Translation.__table__()
cursor = connection.cursor()
cursor_select = connection.cursor(row_factory=backend.dict_row)
for view in views:
cursor.execute(*translation.select(
cursor_select.execute(*translation.select(
translation.id, translation.name, translation.src,
where=(translation.lang == INTERNAL_LANG)
& (translation.type == 'view')
& (translation.name == view.model)
& (translation.module == view.module)))
trans_views = {t['src']: t for t in cursor_dict(cursor)}
trans_views = {t['src']: t for t in cursor_select}
xml = (view.arch or '').strip()
if not xml:
@@ -1574,8 +1584,9 @@ class TranslationUpdate(Wizard):
Translation = pool.get('ir.translation')
Report = pool.get('ir.action.report')
View = pool.get('ir.ui.view')
cursor = Transaction().connection.cursor()
cursor_update = Transaction().connection.cursor()
connection = Transaction().connection
cursor = connection.cursor(row_factory=backend.dict_row)
cursor_update = connection.cursor()
translation = Translation.__table__()
lang = self.start.language.code
parent_lang = get_parent(lang)
@@ -1608,18 +1619,7 @@ class TranslationUpdate(Wizard):
where=(translation.lang == lang)
& source_clause
& translation.type.in_(self._source_types))))
to_create = []
for row in cursor_dict(cursor):
to_create.append({
'name': row['name'],
'res_id': row['res_id'],
'lang': lang,
'type': row['type'],
'src': row['src'],
'src_plural': row['src_plural'],
'module': row['module'],
})
if to_create:
if to_create := list(cursor):
Translation.create(to_create)
if parent_lang:
@@ -1632,7 +1632,7 @@ class TranslationUpdate(Wizard):
where=(translation.lang == lang)
& source_clause
& translation.type.in_(self._source_types))))
for row in cursor_dict(cursor):
for row in cursor:
cursor_update.execute(*translation.update(
[translation.value],
[''],
@@ -1647,8 +1647,11 @@ class TranslationUpdate(Wizard):
if self.model in {Report, View}:
return
columns = [translation.name.as_('name'),
translation.res_id.as_('res_id'), translation.type.as_('type'),
columns = [
translation.name.as_('name'),
translation.res_id.as_('res_id'),
Literal(lang).as_('lang'),
translation.type.as_('type'),
translation.module.as_('module')]
cursor.execute(*(translation.select(*columns,
where=(translation.lang == default_lang)
@@ -1656,16 +1659,7 @@ class TranslationUpdate(Wizard):
- translation.select(*columns,
where=(translation.lang == lang)
& translation.type.in_(self._ressource_types))))
to_create = []
for row in cursor_dict(cursor):
to_create.append({
'name': row['name'],
'res_id': row['res_id'],
'lang': lang,
'type': row['type'],
'module': row['module'],
})
if to_create:
if to_create := list(cursor):
Translation.create(to_create)
if parent_lang:
@@ -1676,7 +1670,7 @@ class TranslationUpdate(Wizard):
& translation.select(*columns,
where=(translation.lang == lang)
& translation.type.in_(self._ressource_types))))
for row in cursor_dict(cursor):
for row in cursor:
cursor_update.execute(*translation.update(
[translation.value],
[''],
@@ -1700,7 +1694,7 @@ class TranslationUpdate(Wizard):
- translation.select(*columns,
where=(translation.lang == lang)
& translation.type.in_(self._updatable_types))))
for row in cursor_dict(cursor):
for row in cursor:
cursor_update.execute(*translation.update(
[translation.fuzzy, translation.src,
translation.src_plural],
@@ -1727,7 +1721,7 @@ class TranslationUpdate(Wizard):
& (translation.value != Null),
group_by=translation.src))
for row in cursor_dict(cursor):
for row in cursor:
cursor_update.execute(*translation.update(
[translation.fuzzy, translation.value],
[True, row['value']],
+10 -7
View File
@@ -21,7 +21,7 @@ from trytond.pool import Pool
from trytond.pyson import PYSONDecoder, PYSONEncoder
from trytond.rpc import RPC
from trytond.sql.functions import Range
from trytond.tools import cursor_dict, grouped_slice
from trytond.tools import grouped_slice
from trytond.tools.domain_inversion import simplify
from trytond.transaction import (
Transaction, inactive_records, record_cache_size, without_check_access)
@@ -997,7 +997,7 @@ class ModelSQL(ModelStorage):
@no_table_query
def create(cls, vlist):
transaction = Transaction()
cursor = transaction.connection.cursor()
cursor = transaction.connection.cursor(row_factory=backend.scalar_row)
pool = Pool()
Translation = pool.get('ir.translation')
@@ -1039,7 +1039,7 @@ class ModelSQL(ModelStorage):
if transaction.database.has_returning():
cursor.execute(*table.insert(
cols, [val], [table.id]))
yield from (r[0] for r in cursor)
yield from cursor
else:
id_new = transaction.database.nextid(
transaction.connection, cls._table)
@@ -1181,7 +1181,7 @@ class ModelSQL(ModelStorage):
Rule = pool.get('ir.rule')
Translation = pool.get('ir.translation')
transaction = Transaction()
cursor = Transaction().connection.cursor()
connection = transaction.connection
ids, fields_names = cls._before_read(ids, fields_names)
@@ -1290,9 +1290,10 @@ class ModelSQL(ModelStorage):
where &= history_clause
if domain:
where &= dom_exp
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*from_.select(*columns.values(), where=where,
order_by=history_order, limit=history_limit))
fetchall = list(cursor_dict(cursor))
fetchall = list(cursor)
if not len(fetchall) == len({}.fromkeys(sub_ids)):
cls.__check_domain_rule(ids, 'read')
raise RuntimeError("Undetected access error")
@@ -1938,7 +1939,7 @@ class ModelSQL(ModelStorage):
def search(cls, domain, offset=0, limit=None, order=None, count=False,
query=False):
transaction = Transaction()
cursor = transaction.connection.cursor()
connection = transaction.connection
super().search(
domain, offset=offset, limit=limit, order=order, count=count)
@@ -1965,6 +1966,7 @@ class ModelSQL(ModelStorage):
if query:
return select
else:
cursor = connection.cursor()
cursor.execute(*select)
return cursor.fetchone()[0]
@@ -2001,9 +2003,10 @@ class ModelSQL(ModelStorage):
if query:
return select
cursor = connection.cursor(row_factory=backend.dict_row)
cursor.execute(*select)
rows = list(cursor_dict(cursor))
rows = list(cursor)
cache = transaction.get_cache()
delete_records = transaction.delete_records[cls.__name__]
-11
View File
@@ -54,20 +54,9 @@ class ClassProperty(property):
return self.fget.__get__(None, owner)()
def cursor_dict(cursor, size=None):
size = cursor.arraysize if size is None else size
while True:
rows = cursor.fetchmany(size)
if not rows:
break
for row in rows:
yield {d[0]: v for d, v in zip(cursor.description, row)}
__all__ = [
ClassProperty,
cached_property,
cursor_dict,
decistmt,
entry_points,
escape_wildcard,