From 5823bce6cf305e82425cb348d5f68e001bcc4708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Thu, 4 Jul 2024 08:45:19 +0200 Subject: [PATCH] Add tools to apply types to SQLite select queries --- modules/account/account.py | 73 +++++++++++----------- modules/account/company.py | 13 ++-- modules/account/journal.py | 17 ++--- modules/account/party.py | 13 ++-- modules/account/tax.py | 11 ++-- modules/account_budget/account.py | 24 +++---- modules/account_deposit/party.py | 24 +++---- modules/account_es/reporting_tax.py | 11 +--- modules/account_invoice/invoice.py | 18 +++--- modules/account_payment/payment.py | 38 ++++++----- modules/account_receivable_rule/account.py | 13 ++-- modules/analytic_account/account.py | 34 +++++----- modules/commission/commission.py | 19 +++--- modules/production_work/work.py | 17 ++--- modules/project_invoice/project.py | 20 +++--- modules/sale_point/sale.py | 12 ++-- trytond/CHANGELOG | 1 + trytond/doc/ref/tools/misc.rst | 4 ++ trytond/trytond/tests/test_tools.py | 14 ++++- trytond/trytond/tools/__init__.py | 5 +- trytond/trytond/tools/misc.py | 11 +++- 21 files changed, 213 insertions(+), 179 deletions(-) diff --git a/modules/account/account.py b/modules/account/account.py index b1168b8f6a..fa86c679d6 100644 --- a/modules/account/account.py +++ b/modules/account/account.py @@ -22,7 +22,8 @@ from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If, PYSONEncoder from trytond.report import Report from trytond.tools import ( - grouped_slice, is_full_text, lstrip_wildcard, reduce_ids) + grouped_slice, is_full_text, lstrip_wildcard, reduce_ids, + sqlite_apply_types) from trytond.transaction import Transaction, check_access, inactive_records from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -1045,21 +1046,22 @@ class Account( line_query, fiscalyear_ids = MoveLine.query_get(line) for sub_ids in grouped_slice(ids): red_sql = reduce_ids(table_a.id, sub_ids) - cursor.execute(*table_a.join(table_c, + query = (table_a.join(table_c, condition=(table_c.left >= table_a.left) & (table_c.right <= table_a.right) ).join(line, condition=line.account == table_c.id ).select( table_a.id, - Sum(Coalesce(line.debit, 0) - Coalesce(line.credit, 0)), + Sum(Coalesce(line.debit, 0) - Coalesce(line.credit, 0) + ).as_('balance'), where=red_sql & line_query, group_by=table_a.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) balances.update(dict(cursor)) for account in accounts: - # SQLite uses float for SUM - if not isinstance(balances[account.id], Decimal): - balances[account.id] = Decimal(str(balances[account.id])) balances[account.id] = account.currency.round(balances[account.id]) fiscalyears = FiscalYear.browse(fiscalyear_ids) @@ -1094,27 +1096,28 @@ class Account( line = MoveLine.__table__() line_query, fiscalyear_ids = MoveLine.query_get(line) columns = [table.id] + types = [None] for name in names: if name == 'line_count': columns.append(Count(Literal('*'))) + types.append(None) else: - columns.append(Sum(Coalesce(Column(line, name), 0))) + columns.append(Sum(Coalesce(Column(line, name), 0)).as_(name)) + types.append('NUMERIC') for sub_ids in grouped_slice(ids): red_sql = reduce_ids(table.id, sub_ids) - cursor.execute(*table.join(line, 'LEFT', + query = (table.join(line, 'LEFT', condition=line.account == table.id ).select(*columns, where=red_sql & line_query, group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, types) + cursor.execute(*query) for row in cursor: account_id = row[0] for i, name in enumerate(names, 1): - # SQLite uses float for SUM - if (name != 'line_count' - and not isinstance(row[i], Decimal)): - result[name][account_id] = Decimal(str(row[i])) - else: - result[name][account_id] = row[i] + result[name][account_id] = row[i] for account in accounts: for name in names: if name == 'line_count': @@ -1541,7 +1544,7 @@ class AccountParty(ActivePeriodMixin, ModelSQL): account_sql = reduce_ids(table_a.id, sub_account_ids) for sub_party_ids in grouped_slice(party_ids): party_sql = reduce_ids(line.party, sub_party_ids) - cursor.execute(*table_a.join(table_c, + query = (table_a.join(table_c, condition=(table_c.left >= table_a.left) & (table_c.right <= table_a.right) ).join(line, condition=line.account == table_c.id @@ -1550,9 +1553,12 @@ class AccountParty(ActivePeriodMixin, ModelSQL): line.party, Sum( Coalesce(line.debit, 0) - - Coalesce(line.credit, 0)), + - Coalesce(line.credit, 0)).as_('balance'), where=account_sql & party_sql & line_query, group_by=[table_a.id, line.party])) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, None, 'NUMERIC']) + cursor.execute(*query) for account_id, party_id, balance in cursor: try: id_ = account_party2id[(account_id, party_id)] @@ -1563,9 +1569,6 @@ class AccountParty(ActivePeriodMixin, ModelSQL): balances[id_] = balance for record in records: - # SQLite uses float for SUM - if not isinstance(balances[record.id], Decimal): - balances[record.id] = Decimal(str(balances[record.id])) balances[record.id] = record.currency.round(balances[record.id]) fiscalyears = FiscalYear.browse(fiscalyear_ids) @@ -1599,20 +1602,26 @@ class AccountParty(ActivePeriodMixin, ModelSQL): line = MoveLine.__table__() line_query, fiscalyear_ids = MoveLine.query_get(line) columns = [table.id, line.party] + types = [None, None] for name in names: if name == 'line_count': - columns.append(Count(Literal('*'))) + columns.append(Count(Literal('*')).as_(name)) + types.append(None) else: - columns.append(Sum(Coalesce(Column(line, name), 0))) + columns.append(Sum(Coalesce(Column(line, name), 0)).as_(name)) + types.append('NUMERIC') for sub_account_ids in grouped_slice(account_ids): account_sql = reduce_ids(table.id, sub_account_ids) for sub_party_ids in grouped_slice(party_ids): party_sql = reduce_ids(line.party, sub_party_ids) - cursor.execute(*table.join(line, 'LEFT', + query = (table.join(line, 'LEFT', condition=line.account == table.id ).select(*columns, where=account_sql & party_sql & line_query, group_by=[table.id, line.party])) + if backend.name == 'sqlite': + sqlite_apply_types(query, types) + cursor.execute(*query) for row in cursor: try: id_ = account_party2id[tuple(row[0:2])] @@ -1621,12 +1630,7 @@ class AccountParty(ActivePeriodMixin, ModelSQL): # the database than from records continue for i, name in enumerate(names, 2): - # SQLite uses float for SUM - if (name != 'line_count' - and not isinstance(row[i], Decimal)): - result[name][id_] = Decimal(str(row[i])) - else: - result[name][id_] = row[i] + result[name][id_] = row[i] for record in records: for name in names: if name == 'line_count': @@ -1759,7 +1763,7 @@ class AccountDeferral(ModelSQL, ModelView): for sub_deferrals in grouped_slice(deferrals): red_sql = reduce_ids(table.id, [d.id for d in sub_deferrals]) - cursor.execute(*table + query = (table .join(account, condition=table.account == account.id) .join(account_child, condition=(account_child.left >= account.left) @@ -1768,15 +1772,14 @@ class AccountDeferral(ModelSQL, ModelView): condition=(table_child.account == account_child.id) & (table_child.fiscalyear == table.fiscalyear)) .select( - table.id, - Sum(table_child.debit - table_child.credit), + table.id.as_('id'), + Sum(table_child.debit - table_child.credit).as_('balance'), where=red_sql, group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) balances.update(dict(cursor)) - - for id_, balance in balances.items(): - if not isinstance(balance, Decimal): - balances[id_] = Decimal(str(balance)) return balances def get_currency(self, name): diff --git a/modules/account/company.py b/modules/account/company.py index 27b090d4cc..b4a2dc2f87 100644 --- a/modules/account/company.py +++ b/modules/account/company.py @@ -6,12 +6,13 @@ from sql import Literal, Null from sql.aggregate import Sum from sql.conditionals import Coalesce +from trytond import backend from trytond.i18n import gettext from trytond.model import fields from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction @@ -67,21 +68,21 @@ class Company(metaclass=PoolMeta): for sub_companies in grouped_slice(companies): sub_ids = [p.id for p in sub_companies] company_where = reduce_ids(account.company, sub_ids) - cursor.execute(*line + query = (line .join(account, condition=account.id == line.account) .join(account_type, condition=account.type == account_type.id) - .select(account.company, amount, + .select(account.company.as_('id'), amount.as_(name), where=(getattr(account_type, code) & (line.reconciliation == Null) & company_where & today_where), group_by=account.company)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) for company_id, value in cursor: - # SQLite uses float for SUM - if not isinstance(value, Decimal): - value = Decimal(str(value)) amounts[name][company_id] = value.quantize(exp[company_id]) return amounts diff --git a/modules/account/journal.py b/modules/account/journal.py index 27a2cc8c0d..eb6533412b 100644 --- a/modules/account/journal.py +++ b/modules/account/journal.py @@ -1,9 +1,9 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from decimal import Decimal from sql.aggregate import Sum +from trytond import backend from trytond.i18n import gettext from trytond.model import ( DeactivableMixin, MatchMixin, ModelSQL, ModelView, Unique, Workflow, @@ -15,7 +15,8 @@ from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id from trytond.tools import ( - grouped_slice, is_full_text, lstrip_wildcard, reduce_ids) + grouped_slice, is_full_text, lstrip_wildcard, reduce_ids, + sqlite_apply_types) from trytond.transaction import Transaction STATES = { @@ -138,16 +139,16 @@ class Journal( ).join(account, 'LEFT', condition=line.account == account.id ).join(account_type, 'LEFT', condition=account.type == account_type.id - ).select(move.journal, Sum(line.debit), Sum(line.credit), + ).select( + move.journal, + Sum(line.debit).as_('debit'), + Sum(line.credit).as_('credit'), where=where & red_sql, group_by=move.journal) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC', 'NUMERIC']) cursor.execute(*query) for journal_id, debit, credit in cursor: - # SQLite uses float for SUM - if not isinstance(debit, Decimal): - debit = Decimal(str(debit)) - if not isinstance(credit, Decimal): - credit = Decimal(str(credit)) result['debit'][journal_id] = company.currency.round(debit) result['credit'][journal_id] = company.currency.round(credit) result['balance'][journal_id] = company.currency.round( diff --git a/modules/account/party.py b/modules/account/party.py index 1a47ae9bea..952edeba9f 100644 --- a/modules/account/party.py +++ b/modules/account/party.py @@ -6,6 +6,7 @@ from sql import Literal, Null from sql.aggregate import Sum from sql.conditionals import Coalesce +from trytond import backend from trytond.i18n import gettext from trytond.model import ModelSQL, fields from trytond.modules.company.model import ( @@ -14,7 +15,7 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.party.exceptions import EraseError from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval, If -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction from .exceptions import AccountMissing @@ -158,21 +159,21 @@ class Party(CompanyMultiValueMixin, metaclass=PoolMeta): for sub_parties in grouped_slice(parties): sub_ids = [p.id for p in sub_parties] party_where = reduce_ids(line.party, sub_ids) - cursor.execute(*line.join(account, + query = (line.join(account, condition=account.id == line.account ).join(account_type, condition=account.type == account_type.id - ).select(line.party, amount, + ).select(line.party, amount.as_(name), where=(getattr(account_type, code) & (line.reconciliation == Null) & (account.company == company_id) & party_where & today_where), group_by=line.party)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) for party, value in cursor: - # SQLite uses float for SUM - if not isinstance(value, Decimal): - value = Decimal(str(value)) result[name][party] = value.quantize(exp) return result diff --git a/modules/account/tax.py b/modules/account/tax.py index 09e5813332..24c00ed7aa 100644 --- a/modules/account/tax.py +++ b/modules/account/tax.py @@ -18,7 +18,8 @@ from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Bool, Eval, If, PYSONEncoder -from trytond.tools import cursor_dict, is_full_text, lstrip_wildcard +from trytond.tools import ( + cursor_dict, is_full_text, lstrip_wildcard, sqlite_apply_types) from trytond.transaction import Transaction from trytond.wizard import Button, StateAction, StateView, Wizard @@ -949,13 +950,13 @@ class Tax(sequence_ordered(), ModelSQL, ModelView, DeactivableMixin): group_by=tax_line.tax) ) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None] + ['NUMERIC'] * len(names)) + cursor.execute(*query) for row in cursor_dict(cursor): for name in names: - value = row[name] or 0 - if not isinstance(value, Decimal): - value = Decimal(str(value)) - result[name][row['tax']] = value + result[name][row['tax']] = row[name] or 0 return result @classmethod diff --git a/modules/account_budget/account.py b/modules/account_budget/account.py index 63f8d80652..d44d371885 100644 --- a/modules/account_budget/account.py +++ b/modules/account_budget/account.py @@ -7,6 +7,7 @@ from itertools import groupby from sql.aggregate import Sum from sql.conditionals import Coalesce +from trytond import backend from trytond.i18n import gettext from trytond.model import ( Index, ModelSQL, ModelView, Unique, fields, sequence_ordered, tree) @@ -15,7 +16,7 @@ from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Bool, Eval, If from trytond.rpc import RPC -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -54,16 +55,15 @@ class AmountMixin: for keys, grouped_records in groupby( records, key=cls._get_amount_group_key): for sub_records in grouped_slice(list(grouped_records)): - cursor.execute(*cls._get_amount_query( - sub_records, dict(keys))) + query = cls._get_amount_query(sub_records, dict(keys)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) balances.update(cursor) total = {n: {} for n in names} for record in records: balance = balances[record.id] - # SQLite uses float for SUM - if not isinstance(balance, Decimal): - balance = Decimal(str(balance)) if 'actual_amount' in names: total['actual_amount'][record.id] = record.currency.round( balance) @@ -205,9 +205,12 @@ class BudgetLineMixin( & (children.right <= table.right)) .select( table.id, - Sum(Coalesce(children.amount, 0)), + Sum(Coalesce(children.amount, 0)).as_(name), group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + for sub_ids in grouped_slice(ids): query.where = reduce_ids(table.id, sub_ids) cursor.execute(*query) @@ -215,9 +218,6 @@ class BudgetLineMixin( for record in records: amount = amounts[record.id] - # SQLite uses float for SUM - if amount is not None and not isinstance(amount, Decimal): - amount = Decimal(str(amount)) if amount is not None: amounts[record.id] = record.currency.round(amount) return amounts @@ -564,7 +564,7 @@ class BudgetLine(BudgetLineMixin, ModelSQL, ModelView): line, condition=line.account == children.account) .select( - table.id, amount, + table.id, amount.as_('amount'), where=red_sql & query_where, group_by=table.id)) @@ -714,7 +714,7 @@ class BudgetLinePeriod(AmountMixin, ModelSQL, ModelView): condition=(line.move == move.id) & (move.period == table.period)) .select( - table.id, amount, + table.id, amount.as_('amount'), where=red_sql & query_where, group_by=table.id)) diff --git a/modules/account_deposit/party.py b/modules/account_deposit/party.py index 2d0de49148..87780bcdb7 100644 --- a/modules/account_deposit/party.py +++ b/modules/account_deposit/party.py @@ -6,12 +6,13 @@ from sql import For, Literal, Null from sql.aggregate import Sum from sql.conditionals import Case, Coalesce +from trytond import backend from trytond.i18n import gettext from trytond.model import fields from trytond.modules.currency.fields import Monetary from trytond.modules.party.exceptions import EraseError from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction @@ -46,23 +47,23 @@ class Party(metaclass=PoolMeta): for sub_parties in grouped_slice(parties): party_clause = reduce_ids(line.party, [p.id for p in sub_parties]) - cursor.execute(*line + query = (line .join(account, condition=account.id == line.account) .join(account_type, condition=account.type == account_type.id) .select(line.party, # Use credit - debit to positive deposit amount - Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0)), + Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0) + ).as_('debit'), where=account_type.deposit & party_clause & (line.reconciliation == Null) & (account.company == user.company.id) & line_clause, group_by=line.party)) - for party_id, value in cursor: - # SQLite uses float for SUM - if not isinstance(value, Decimal): - value = currency.round(Decimal(str(value))) - values[party_id] = value + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + values.update((p, currency.round(a)) for p, a in cursor) return values @classmethod @@ -131,12 +132,13 @@ class Party(metaclass=PoolMeta): line.amount_second_currency), else_=0)) - cursor.execute(*line.select(amount, where=where)) + query = line.select(amount.as_('deposit_balance'), where=where) + if backend.name == 'sqlite': + sqlite_apply_types(query, ['NUMERIC']) + cursor.execute(*query) amount, = cursor.fetchone() if amount is None: amount = Decimal(0) - if not isinstance(amount, Decimal): - amount = Decimal(str(amount)) return currency.round(amount) def check_deposit(self, deposit_account, sign=1): diff --git a/modules/account_es/reporting_tax.py b/modules/account_es/reporting_tax.py index 4276fba425..60fda8fdc8 100644 --- a/modules/account_es/reporting_tax.py +++ b/modules/account_es/reporting_tax.py @@ -53,8 +53,6 @@ def justify(string, size): def format_decimal(n, include_sign=False): - if not isinstance(n, Decimal): - n = Decimal(n) sign = '' if include_sign: sign = 'N' if n < 0 else '' @@ -470,7 +468,7 @@ class ESVATList(ModelSQL, ModelView): exclude_invoice_tax.invoice, where=((exclude_invoice_tax.invoice == invoice.id) & (exclude_invoice_tax.tax.in_(excluded_taxes)))))) - return (tax_line + query = (tax_line .join(tax, condition=tax_line.tax == tax.id) .join(line, condition=tax_line.move_line == line.id) .join(move, condition=line.move == move.id) @@ -508,6 +506,7 @@ class ESVATList(ModelSQL, ModelView): company.currency, tax.es_vat_list_code, ])) + return query class ESVATListContext(ModelView): @@ -548,8 +547,6 @@ class AEAT347(Report): context['justify'] = justify def format_decimal(n): - if not isinstance(n, Decimal): - n = Decimal(n) sign = 'N' if n < 0 else ' ' return sign + ('{0:.2f}'.format(abs(n))).replace('.', '').rjust( 15, '0') @@ -707,8 +704,6 @@ class AEAT349(Report): context['identifier_code'] = identifier_code def format_decimal(n, digits=13): - if not isinstance(n, Decimal): - n = Decimal(n) return ('{0:.2f}'.format(abs(n))).replace('.', '').rjust( digits, '0') context['format_decimal'] = format_decimal @@ -928,8 +923,6 @@ class VATBookReport(Report): def format_decimal(cls, n): if n is None: return '' - if not isinstance(n, Decimal): - n = Decimal(n) sign = '-' if n < 0 else '' return sign + '{0:.2f}'.format(abs(n)).replace('.', ',') diff --git a/modules/account_invoice/invoice.py b/modules/account_invoice/invoice.py index 618f18c4dd..9ff1db0a33 100644 --- a/modules/account_invoice/invoice.py +++ b/modules/account_invoice/invoice.py @@ -30,7 +30,8 @@ from trytond.pyson import Bool, Eval, Id, If from trytond.report import Report from trytond.rpc import RPC from trytond.tools import ( - cached_property, firstline, grouped_slice, reduce_ids, slugify) + cached_property, firstline, grouped_slice, reduce_ids, slugify, + sqlite_apply_types) from trytond.transaction import Transaction from trytond.wizard import ( Button, StateAction, StateReport, StateTransition, StateView, Wizard) @@ -715,21 +716,18 @@ class Invoice(Workflow, ModelSQL, ModelView, TaxableMixin, InvoiceReportMixin): type_name = cls.tax_amount._field.sql_type().base tax = InvoiceTax.__table__() - to_round = False for sub_ids in grouped_slice(invoices_no_cache): red_sql = reduce_ids(tax.invoice, sub_ids) - cursor.execute(*tax.select(tax.invoice, + query = (tax.select(tax.invoice, Coalesce(Sum(tax.amount), 0).as_(type_name), where=red_sql, group_by=tax.invoice)) - for invoice_id, sum_ in cursor: - # SQLite uses float for SUM - if not isinstance(sum_, Decimal): - sum_ = Decimal(str(sum_)) - to_round = True - tax_amount[invoice_id] = sum_ + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + tax_amount.update(cursor) # Float amount must be rounded to get the right precision - if to_round: + if backend.name == 'sqlite': for invoice in invoices: tax_amount[invoice.id] = invoice.currency.round( tax_amount[invoice.id]) diff --git a/modules/account_payment/payment.py b/modules/account_payment/payment.py index 0adc2599ba..d9dec7d5f0 100644 --- a/modules/account_payment/payment.py +++ b/modules/account_payment/payment.py @@ -8,6 +8,7 @@ from sql.aggregate import Count, Sum from sql.functions import CharLength from sql.operators import Abs +from trytond import backend from trytond.i18n import gettext from trytond.model import ( DeactivableMixin, Index, ModelSQL, ModelView, Workflow, fields) @@ -19,7 +20,8 @@ from trytond.pool import Pool from trytond.pyson import Eval, If from trytond.rpc import RPC from trytond.tools import ( - cursor_dict, grouped_slice, reduce_ids, sortable_values) + cursor_dict, grouped_slice, reduce_ids, sortable_values, + sqlite_apply_types) from trytond.transaction import Transaction from trytond.wizard import StateAction, Wizard @@ -196,34 +198,30 @@ class Group(ModelSQL, ModelView): ] for sub_ids in grouped_slice(groups): - cursor.execute(*payment.select(*columns, + query = payment.select(*columns, where=reduce_ids(payment.group, sub_ids), - group_by=payment.group), - ) - + group_by=payment.group) + if backend.name == 'sqlite': + sqlite_apply_types( + query, [None, None, 'NUMERIC', 'NUMERIC', None]) + cursor.execute(*query) for row in cursor_dict(cursor): - group_id = row['group_id'] + group = cls(row['group_id']) - result['payment_count'][group_id] = row['payment_count'] - result['payment_complete'][group_id] = \ + result['payment_count'][group.id] = row['payment_count'] + result['payment_complete'][group.id] = \ not row['payment_not_complete'] amount = row['payment_amount'] succeeded = row['payment_amount_succeeded'] - if amount is not None: - # SQLite uses float for SUM - if not isinstance(amount, Decimal): - amount = Decimal(str(amount)) - amount = cls(group_id).company.currency.round(amount) - result['payment_amount'][group_id] = amount + if amount is not None and backend.name == 'sqlite': + amount = group.company.currency.round(amount) + result['payment_amount'][group.id] = amount - if succeeded is not None: - # SQLite uses float for SUM - if not isinstance(succeeded, Decimal): - succeeded = Decimal(str(succeeded)) - succeeded = cls(group_id).company.currency.round(succeeded) - result['payment_amount_succeeded'][group_id] = succeeded + if succeeded is not None and backend.name == 'sqlite': + succeeded = group.company.currency.round(succeeded) + result['payment_amount_succeeded'][group.id] = succeeded for key in list(result.keys()): if key not in names: diff --git a/modules/account_receivable_rule/account.py b/modules/account_receivable_rule/account.py index 36d1cf6a51..680a1ab4b5 100644 --- a/modules/account_receivable_rule/account.py +++ b/modules/account_receivable_rule/account.py @@ -1,17 +1,18 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. import datetime as dt -from decimal import Decimal from sql import Literal, Null from sql.aggregate import Sum from sql.operators import Equal +from trytond import backend from trytond.model import ( DeactivableMixin, Exclude, ModelSQL, ModelView, Unique, Workflow, dualmethod, fields, sequence_ordered) from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction @@ -242,21 +243,21 @@ class AccountRuleAbstract(DeactivableMixin, ModelSQL, ModelView): cursor = Transaction().connection.cursor() amount = Sum(self._amount(line)) - cursor.execute(*line + query = (line .join(move, condition=line.move == move.id) .select( line.party, - amount, + amount.as_('amount'), where=( (line.account == self.account.id) & (line.reconciliation == Null) & (move.state == 'posted')), group_by=line.party, having=amount > 0)) - + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) for party_id, amount in cursor: - if not isinstance(amount, Decimal): - amount = Decimal(str(amount)) if amount > 0: yield party_id, amount diff --git a/modules/analytic_account/account.py b/modules/analytic_account/account.py index d004b5b874..8972ce3959 100644 --- a/modules/analytic_account/account.py +++ b/modules/analytic_account/account.py @@ -7,6 +7,7 @@ from sql import Column, Literal from sql.aggregate import Sum from sql.conditionals import Coalesce +from trytond import backend from trytond.i18n import gettext from trytond.model import ( DeactivableMixin, Index, ModelSQL, ModelView, Unique, fields, sum_tree, @@ -15,7 +16,8 @@ from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Eval, If, PYSONDecoder, PYSONEncoder -from trytond.tools import grouped_slice, is_full_text, lstrip_wildcard +from trytond.tools import ( + grouped_slice, is_full_text, lstrip_wildcard, sqlite_apply_types) from trytond.transaction import Transaction from .exceptions import AccountValidationError @@ -195,22 +197,22 @@ class Account( id2account[account.id] = account line_query = Line.query_get(line) - cursor.execute(*table.join(line, 'LEFT', + query = (table.join(line, 'LEFT', condition=table.id == line.account ).join(move_line, 'LEFT', condition=move_line.id == line.move_line ).select(table.id, - Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0)), + Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0) + ).as_('balance'), where=(table.type != 'view') & table.id.in_(all_ids) & (table.active == Literal(True)) & line_query, group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) values = defaultdict(Decimal) - for account_id, value in cursor: - # SQLite uses float for SUM - if not isinstance(value, Decimal): - value = Decimal(str(value)) - values[account_id] += value + values.update(cursor) balances = sum_tree(childs, values) for account in accounts: @@ -240,9 +242,11 @@ class Account( line_query = Line.query_get(line) columns = [table.id] + types = [None] for name in names: - columns.append(Sum(Coalesce(Column(line, name), 0))) - cursor.execute(*table.join(line, 'LEFT', + columns.append(Sum(Coalesce(Column(line, name), 0)).as_(name)) + types.append('NUMERIC') + query = (table.join(line, 'LEFT', condition=table.id == line.account ).join(move_line, 'LEFT', condition=move_line.id == line.move_line @@ -251,15 +255,13 @@ class Account( & table.id.in_(ids) & (table.active == Literal(True)) & line_query, group_by=table.id)) - + if backend.name == 'sqlite': + sqlite_apply_types(query, types) + cursor.execute(*query) for row in cursor: account_id = row[0] for i, name in enumerate(names, 1): - value = row[i] - # SQLite uses float for SUM - if not isinstance(value, Decimal): - value = Decimal(str(value)) - result[name][account_id] += value + result[name][account_id] += row[i] for account in accounts: for name in names: result[name][account.id] = account.currency.round( diff --git a/modules/commission/commission.py b/modules/commission/commission.py index 46ffc9f73f..e4d3002f40 100644 --- a/modules/commission/commission.py +++ b/modules/commission/commission.py @@ -12,6 +12,7 @@ except ImportError: Null = None from sql.aggregate import Sum +from trytond import backend from trytond.i18n import gettext from trytond.model import ( MatchMixin, ModelSQL, ModelView, fields, sequence_ordered) @@ -19,7 +20,8 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import decistmt, grouped_slice, reduce_ids +from trytond.tools import ( + decistmt, grouped_slice, reduce_ids, sqlite_apply_types) from trytond.transaction import Transaction, check_access from trytond.wizard import Button, StateAction, StateView, Wizard @@ -122,17 +124,18 @@ class Agent(ModelSQL, ModelView): for sub_ids in grouped_slice(ids): where = reduce_ids(commission.agent, sub_ids) where &= commission.invoice_line == Null - query = commission.select(commission.agent, Sum(commission.amount), + query = commission.select( + commission.agent, Sum(commission.amount).as_('pending_amount'), where=where, group_by=commission.agent) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) cursor.execute(*query) amounts.update(dict(cursor)) - for agent_id, amount in amounts.items(): - if amount: - # SQLite uses float for SUM - if not isinstance(amount, Decimal): - amount = Decimal(str(amount)) - amounts[agent_id] = round_price(amount) + if backend.name == 'sqlite': + for agent_id, amount in amounts.items(): + if amount is not None: + amounts[agent_id] = round_price(amount) return amounts @property diff --git a/modules/production_work/work.py b/modules/production_work/work.py index 8a38666398..a89a301b57 100644 --- a/modules/production_work/work.py +++ b/modules/production_work/work.py @@ -9,6 +9,7 @@ from functools import wraps from sql.aggregate import Sum from sql.conditionals import Coalesce +from trytond import backend from trytond.i18n import gettext from trytond.model import ( DeactivableMixin, Index, ModelSQL, ModelView, Workflow, fields, @@ -18,7 +19,7 @@ from trytond.modules.company.model import employee_field, set_employee from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, If, TimeDelta -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction from .exceptions import PickerError @@ -262,15 +263,15 @@ class Work(sequence_ordered(), ModelSQL, ModelView): for sub_works in grouped_slice(works): red_sql = reduce_ids(cycle.work, [w.id for w in sub_works]) - cursor.execute(*cycle.select( - cycle.work, Sum(Coalesce(cycle.cost, 0)), - where=red_sql & (cycle.state == 'done'), - group_by=cycle.work)) + query = cycle.select( + cycle.work, Sum(Coalesce(cycle.cost, 0)).as_('cost'), + where=red_sql & (cycle.state == 'done'), + group_by=cycle.work) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) costs.update(cursor) - for cost in costs: - if not isinstance(cost, Decimal): - costs[cost] = Decimal(str(costs[cost])) costs[cost] = round_price(costs[cost]) return costs diff --git a/modules/project_invoice/project.py b/modules/project_invoice/project.py index 3e09ab58d9..f8b58dc0b5 100644 --- a/modules/project_invoice/project.py +++ b/modules/project_invoice/project.py @@ -7,19 +7,20 @@ from collections import defaultdict from decimal import Decimal from itertools import groupby -from sql import Null +from sql import Cast, Null from sql.aggregate import Sum from sql.conditionals import NullIf from sql.functions import Extract from sql.operators import Concat +from trytond import backend from trytond.i18n import gettext from trytond.model import ModelSQL, ModelView, fields from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import StateAction, Wizard @@ -164,17 +165,20 @@ class Progress: ids2work = dict((w.id, w) for w in works) for sub_ids in grouped_slice(ids2work.keys()): where = reduce_ids(table.id, sub_ids) - cursor.execute(*table.join(progress, + query = (table.join(progress, condition=progress.work == table.id ).join(invoice_line, condition=progress.invoice_line == invoice_line.id - ).select(table.id, - Sum(progress.progress * invoice_line.unit_price), + ).select( + table.id, + Sum(Cast(progress.progress, 'NUMERIC') + * invoice_line.unit_price).as_('amount'), where=where, group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) for work_id, amount in cursor: - if not isinstance(amount, Decimal): - amount = Decimal(str(amount)) work = ids2work[work_id] if work.price_list_hour: amount *= Decimal(str(work.effort_hours)) @@ -191,7 +195,7 @@ class Progress: for work in works: currency = id2currency[work2currency[work.id]] - amounts[work.id] = currency.round(Decimal(amounts[work.id])) + amounts[work.id] = currency.round(amounts[work.id]) return amounts def get_origins_to_invoice(self): diff --git a/modules/sale_point/sale.py b/modules/sale_point/sale.py index e153252891..734d93581c 100644 --- a/modules/sale_point/sale.py +++ b/modules/sale_point/sale.py @@ -9,6 +9,7 @@ from sql.conditionals import Coalesce from sql.functions import CharLength from sql.operators import Equal +from trytond import backend from trytond.i18n import gettext from trytond.model import ( DeactivableMixin, Exclude, Index, ModelSQL, ModelView, Workflow, fields) @@ -18,7 +19,7 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import grouped_slice, reduce_ids +from trytond.tools import grouped_slice, reduce_ids, sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import Button, StateTransition, StateView, Wizard @@ -861,13 +862,12 @@ class POSCashSession(Workflow, ModelSQL, ModelView): group_by=[transfer.session]), all_=True) query = query.select( - query.session, Sum(query.amount), + query.session, Sum(query.amount).as_('amount'), group_by=[query.session]) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) cursor.execute(*query) - for session_id, balance in cursor: - if not isinstance(balance, Decimal): - balance = Decimal(str(balance)) - balances[session_id] = balance + balances.update(cursor) for session in sessions: balances[session.id] = session.currency.round(balances[session.id]) return balances diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG index 8816394b89..a051ed631f 100644 --- a/trytond/CHANGELOG +++ b/trytond/CHANGELOG @@ -1,3 +1,4 @@ +* Add tools to apply types to SQLite select queries * Compress RPC responses * Change _changed_values and _save_values properties to method * Add configuration for report convert command diff --git a/trytond/doc/ref/tools/misc.rst b/trytond/doc/ref/tools/misc.rst index fa531aec8b..1b2dbf8dac 100644 --- a/trytond/doc/ref/tools/misc.rst +++ b/trytond/doc/ref/tools/misc.rst @@ -64,6 +64,10 @@ Miscellaneous Return an SQL expression that pairs SQL integers x and y. +.. function:: sqlite_apply_types(select, types) + + Apply in place the SQLite types to column names of the select query. + .. function:: firstline(text) Return first non-empty line of a text field. diff --git a/trytond/trytond/tests/test_tools.py b/trytond/trytond/tests/test_tools.py index f3499dcfa5..f58ccc1acc 100644 --- a/trytond/trytond/tests/test_tools.py +++ b/trytond/trytond/tests/test_tools.py @@ -24,8 +24,8 @@ from trytond.tools import ( cached_property, decimal_, email_, escape_wildcard, file_open, firstline, grouped_slice, is_full_text, is_instance_method, likify, lstrip_wildcard, pairwise_longest, reduce_domain, reduce_ids, remove_forbidden_chars, - rstrip_wildcard, slugify, sortable_values, strip_wildcard, timezone, - unescape_wildcard) + rstrip_wildcard, slugify, sortable_values, sqlite_apply_types, + strip_wildcard, timezone, unescape_wildcard) from trytond.tools.domain_inversion import ( canonicalize, concat, domain_inversion, eval_domain, extract_reference_models, localize_domain, merge, parse, @@ -330,6 +330,16 @@ class ToolsTestCase(TestCase): (1, None), ]) + def test_sqlite_apply_types(self): + "test sqlite_apply_types" + table = sql.Table('t') + select = table.select(table.foo.as_('bar')) + + sqlite_apply_types(select, ['NUMERIC']) + + self.assertEqual( + str(select), 'SELECT "a"."foo" AS "bar [NUMERIC]" FROM "t" AS "a"') + def test_firstline(self): "Test firstline" for text, result in [ diff --git a/trytond/trytond/tools/__init__.py b/trytond/trytond/tools/__init__.py index 58de092665..03009a0c7c 100644 --- a/trytond/trytond/tools/__init__.py +++ b/trytond/trytond/tools/__init__.py @@ -15,8 +15,8 @@ from .misc import ( get_smtp_server, grouped_slice, import_module, is_full_text, is_instance_method, likify, lstrip_wildcard, pairwise_longest, reduce_domain, reduce_ids, remove_forbidden_chars, resolve, - rstrip_wildcard, slugify, sortable_values, sql_pairing, strip_wildcard, - unescape_wildcard) + rstrip_wildcard, slugify, sortable_values, sql_pairing, sqlite_apply_types, + strip_wildcard, unescape_wildcard) _NOT_FOUND = object() @@ -92,6 +92,7 @@ __all__ = [ slugify, sortable_values, sql_pairing, + sqlite_apply_types, strip_wildcard, unescape_wildcard, ] diff --git a/trytond/trytond/tools/misc.py b/trytond/trytond/tools/misc.py index a66f56c33c..5c49b89c49 100644 --- a/trytond/trytond/tools/misc.py +++ b/trytond/trytond/tools/misc.py @@ -16,7 +16,7 @@ from collections.abc import Iterable, Sized from functools import wraps from itertools import chain, islice, tee, zip_longest -from sql import Literal +from sql import As, Literal, Select from sql.conditionals import Case from sql.operators import Or @@ -306,6 +306,15 @@ def sql_pairing(x, y): else_=(x * x) + x + y) +def sqlite_apply_types(select, types): + "Apply SQLite types to column names of the select query" + assert isinstance(select, Select) + for column, type in zip(select.columns, types): + if type: + assert isinstance(column, As) + column.output_name += ' [%s]' % type + + def firstline(text): "Returns first non-empty line" try: