From 5310b00496293f62a433a32eb6999dfe4d5c50ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Sun, 22 Mar 2026 21:05:26 +0100 Subject: [PATCH] Remove grouped_slice per IN_MAX The usage of ``grouped_slice`` without size is deprecated since `= ANY(ARRAY[%s])` is used for PostgreSQL and SQLite is used only for tests. The ``Database.IN_MAX`` is replaced by ``backend.MAX_QUERY_PARAMS`` which is a more accurate limit of the backend. The SQLite limits is indeed SQLITE_MAX_EXPR_DEPTH so we use a empirical value. --- modules/account/account.py | 281 ++++++------ modules/account/company.py | 40 +- modules/account/journal.py | 66 ++- modules/account/move.py | 242 +++++----- modules/account/party.py | 17 +- modules/account/period.py | 4 +- modules/account_asset/asset.py | 14 +- modules/account_budget/account.py | 45 +- modules/account_deposit/party.py | 41 +- modules/account_dunning_letter/dunning.py | 26 +- modules/account_es_sii/account.py | 17 +- modules/account_invoice/account.py | 26 +- modules/account_invoice/invoice.py | 102 ++--- modules/account_invoice_history/account.py | 45 +- modules/account_payment/account.py | 10 +- modules/account_payment/payment.py | 46 +- modules/account_payment_clearing/account.py | 9 +- modules/account_payment_clearing/payment.py | 14 +- modules/account_payment_sepa/payment.py | 48 +- modules/account_statement/bank.py | 25 +- modules/account_statement/journal.py | 65 +-- modules/analytic_account/account.py | 42 +- modules/commission/commission.py | 27 +- modules/commission/invoice.py | 43 +- modules/customs/product.py | 23 +- .../marketing_automation.py | 58 ++- modules/marketing_email/marketing.py | 40 +- modules/product_price_list_cache/product.py | 25 +- modules/production_work/work.py | 25 +- modules/project_invoice/invoice.py | 21 +- modules/project_invoice/project.py | 156 ++++--- modules/project_revenue/work.py | 67 ++- modules/purchase/product.py | 21 +- modules/purchase_history/purchase.py | 12 +- modules/sale/sale_reporting.py | 32 +- modules/sale_complaint/complaint.py | 4 +- modules/sale_credit_limit/party.py | 70 ++- modules/sale_history/sale.py | 12 +- modules/sale_point/sale.py | 64 ++- modules/sale_promotion_coupon/sale.py | 13 +- modules/sale_stock_quantity/sale.py | 54 ++- modules/sale_subscription_asset/stock.py | 29 +- modules/sale_supply_drop_shipment/product.py | 22 +- modules/sale_supply_production/production.py | 9 +- modules/stock/inventory.py | 11 +- modules/stock/location.py | 39 +- modules/stock/message.xml | 3 - modules/stock/move.py | 35 +- modules/stock/period.py | 14 +- modules/stock/product.py | 88 ++-- modules/stock/stock_reporting_margin.py | 16 +- modules/stock_forecast/forecast.py | 50 +-- modules/stock_lot/stock.py | 28 +- modules/stock_lot_sled/stock.py | 33 +- modules/stock_shipment_measurements/stock.py | 29 +- modules/stock_supply/purchase_request.py | 104 +++-- modules/stock_supply/shipment.py | 14 +- modules/stock_supply_production/production.py | 104 +++-- modules/timesheet/work.py | 24 +- modules/web_shop_shopify/product.py | 18 +- modules/web_shortener/web.py | 16 +- trytond-gis/trytond_gis/postgis/__init__.py | 9 +- trytond/CHANGELOG | 2 + trytond/doc/ref/backend.rst | 4 + trytond/trytond/admin.py | 3 +- trytond/trytond/backend/__init__.py | 3 +- trytond/trytond/backend/database.py | 1 - .../trytond/backend/postgresql/__init__.py | 13 +- .../trytond/backend/postgresql/database.py | 11 +- trytond/trytond/backend/sqlite/__init__.py | 13 +- trytond/trytond/backend/sqlite/database.py | 1 - trytond/trytond/ir/cron.py | 20 +- trytond/trytond/ir/message.py | 27 +- trytond/trytond/ir/model.py | 96 ++-- trytond/trytond/ir/module.py | 18 +- trytond/trytond/ir/note.py | 35 +- trytond/trytond/ir/queue_.py | 7 +- trytond/trytond/ir/resource.py | 8 +- trytond/trytond/ir/rule.py | 16 +- trytond/trytond/ir/translation.py | 64 +-- trytond/trytond/ir/trigger.py | 63 ++- trytond/trytond/ir/ui/menu.py | 23 +- trytond/trytond/model/fields/binary.py | 27 +- trytond/trytond/model/fields/dict.py | 9 +- trytond/trytond/model/fields/field.py | 3 +- trytond/trytond/model/fields/fmany2one.py | 4 +- trytond/trytond/model/fields/many2many.py | 53 +-- trytond/trytond/model/fields/one2many.py | 39 +- trytond/trytond/model/modelsql.py | 424 ++++++++---------- trytond/trytond/model/modelstorage.py | 172 ++++--- trytond/trytond/modules/__init__.py | 8 +- trytond/trytond/res/group.py | 10 +- trytond/trytond/res/user.py | 26 +- trytond/trytond/tests/test_history.py | 25 +- trytond/trytond/tests/test_modelsql.py | 10 - trytond/trytond/tests/test_modelstorage.py | 23 - trytond/trytond/tests/test_tools.py | 25 +- trytond/trytond/tools/misc.py | 6 +- 98 files changed, 1799 insertions(+), 2180 deletions(-) diff --git a/modules/account/account.py b/modules/account/account.py index 6f17f816ac..3257baf87a 100644 --- a/modules/account/account.py +++ b/modules/account/account.py @@ -22,8 +22,7 @@ from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If, PYSONEncoder from trytond.report import Report from trytond.tools import ( - grouped_slice, is_full_text, lstrip_wildcard, pair, sql_pairing, - sqlite_apply_types) + is_full_text, lstrip_wildcard, pair, sql_pairing, sqlite_apply_types) from trytond.transaction import Transaction, check_access, inactive_records from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -1018,27 +1017,26 @@ class Account( ids = [a.id for a in c_accounts] with transaction.set_context(company=company.id): line_query, fiscalyear_ids = MoveLine.query_get(line) - for sub_ids in grouped_slice(ids): - red_sql = fields.SQL_OPERATORS['in'](table_a.id, sub_ids) - if context.get('flat_balance'): - query = (table_a - .join(line, condition=line.account == table_a.id)) - else: - query = (table_a - .join(table_c, - condition=(table_c.left >= table_a.left) - & (table_c.right <= table_a.right)) - .join(line, condition=line.account == table_c.id)) - query = query.select( - table_a.id, - Sum(Coalesce(line.debit, 0) - Coalesce(line.credit, 0) - ).as_('balance'), - where=red_sql & line_query, - group_by=table_a.id) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - balances.update(dict(cursor)) + red_sql = fields.SQL_OPERATORS['in'](table_a.id, ids) + if context.get('flat_balance'): + query = (table_a + .join(line, condition=line.account == table_a.id)) + else: + query = (table_a + .join(table_c, + condition=(table_c.left >= table_a.left) + & (table_c.right <= table_a.right)) + .join(line, condition=line.account == table_c.id)) + query = query.select( + table_a.id, + Sum(Coalesce(line.debit, 0) - Coalesce(line.credit, 0) + ).as_('balance'), + where=red_sql & line_query, + group_by=table_a.id) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + balances.update(dict(cursor)) for account in c_accounts: balances[account.id] = account.currency.round( @@ -1091,20 +1089,19 @@ class Account( columns.append( Sum(Coalesce(Column(line, name), 0)).as_(name)) types.append('NUMERIC') - for sub_ids in grouped_slice(ids): - red_sql = fields.SQL_OPERATORS['in'](table.id, sub_ids) - query = (table.join(line, 'LEFT', - condition=line.account == table.id - ).select(*columns, - where=red_sql & line_query, - group_by=table.id)) - if backend.name == 'sqlite': - sqlite_apply_types(query, types) - cursor.execute(*query) - for row in cursor: - account_id = row[0] - for i, name in enumerate(names, 1): - result[name][account_id] = row[i] + where = fields.SQL_OPERATORS['in'](table.id, ids) + query = (table.join(line, 'LEFT', + condition=line.account == table.id + ).select(*columns, + where=where & line_query, + group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, types) + cursor.execute(*query) + for row in cursor: + account_id = row[0] + for i, name in enumerate(names, 1): + result[name][account_id] = row[i] for account in c_accounts: for name in names: if name == 'line_count': @@ -1161,14 +1158,12 @@ class Account( if fiscalyear.state == 'closed' and deferral: Deferral = pool.get(deferral) id2deferral = {} - ids = [a.id for a in accounts] - for sub_ids in grouped_slice(ids): - deferrals = Deferral.search([ - ('fiscalyear', '=', fiscalyear.id), - ('account', 'in', list(sub_ids)), - ]) - for deferral in deferrals: - id2deferral[deferral.account.id] = deferral + deferrals = Deferral.search([ + ('fiscalyear', '=', fiscalyear), + ('account', 'in', accounts), + ]) + for deferral in deferrals: + id2deferral[deferral.account.id] = deferral for account in accounts: if account.id in id2deferral: @@ -1246,26 +1241,24 @@ class Account( if field_names and not (field_names & {'closed', 'type'}): return accounts = [a for a in accounts if a.closed or not a.type] - for sub_accounts in grouped_slice(accounts): - sub_accounts = list(sub_accounts) - if Line.search([ - ('account', 'in', [a.id for a in sub_accounts]), - ], order=[], limit=1): - for account in sub_accounts: - if not account.closed: - continue - lines = Line.search([ - ('account', '=', account.id), - ], order=[], limit=1) - if lines: - if account.closed: - raise AccountValidationError(gettext( - 'account.msg_account_closed_lines', - account=account.rec_name)) - elif account.type: - raise AccountValidationError(gettext( - 'account.msg_account_no_type_lines', - account=account.rec_name)) + if Line.search([ + ('account', 'in', accounts), + ], order=[], limit=1): + for account in accounts: + if not account.closed: + continue + lines = Line.search([ + ('account', '=', account.id), + ], order=[], limit=1) + if lines: + if account.closed: + raise AccountValidationError(gettext( + 'account.msg_account_closed_lines', + account=account.rec_name)) + elif account.type: + raise AccountValidationError(gettext( + 'account.msg_account_no_type_lines', + account=account.rec_name)) @classmethod def copy(cls, accounts, default=None): @@ -1530,34 +1523,30 @@ class AccountParty(ActivePeriodMixin, ModelSQL): party_ids = {a.party.id for a in c_records} with transaction.set_context(company=company.id): line_query, fiscalyear_ids = MoveLine.query_get(line) - for sub_account_ids in grouped_slice(account_ids): - account_sql = fields.SQL_OPERATORS['in']( - table_a.id, sub_account_ids) - for sub_party_ids in grouped_slice(party_ids): - party_sql = fields.SQL_OPERATORS['in']( - line.party, sub_party_ids) - if context.get('flat_balance'): - query = (table_a - .join(line, condition=line.account == table_a.id)) - else: - query = (table_a - .join(table_c, - condition=(table_c.left >= table_a.left) - & (table_c.right <= table_a.right)) - .join(line, condition=line.account == table_c.id)) - query = query.select( - table_a.id, - line.party, - Sum( - Coalesce(line.debit, 0) - - Coalesce(line.credit, 0)).as_('balance'), - where=account_sql & party_sql & line_query, - group_by=[table_a.id, line.party]) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, None, 'NUMERIC']) - cursor.execute(*query) - for account_id, party_id, balance in cursor: - balances[pair(party_id, account_id)] = balance + account_sql = fields.SQL_OPERATORS['in'](table_a.id, account_ids) + party_sql = fields.SQL_OPERATORS['in'](line.party, party_ids) + if context.get('flat_balance'): + query = (table_a + .join(line, condition=line.account == table_a.id)) + else: + query = (table_a + .join(table_c, + condition=(table_c.left >= table_a.left) + & (table_c.right <= table_a.right)) + .join(line, condition=line.account == table_c.id)) + query = query.select( + table_a.id, + line.party, + Sum( + Coalesce(line.debit, 0) + - Coalesce(line.credit, 0)).as_('balance'), + where=account_sql & party_sql & line_query, + group_by=[table_a.id, line.party]) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, None, 'NUMERIC']) + cursor.execute(*query) + for account_id, party_id, balance in cursor: + balances[pair(party_id, account_id)] = balance for record in c_records: balances[record.id] = record.currency.round( @@ -1609,24 +1598,20 @@ class AccountParty(ActivePeriodMixin, ModelSQL): with transaction.set_context(company=company.id): line_query, fiscalyear_ids = MoveLine.query_get(line) - for sub_account_ids in grouped_slice(account_ids): - account_sql = fields.SQL_OPERATORS['in']( - table.id, sub_account_ids) - for sub_party_ids in grouped_slice(party_ids): - party_sql = fields.SQL_OPERATORS['in']( - line.party, sub_party_ids) - query = (table.join(line, 'LEFT', - condition=line.account == table.id - ).select(*columns, - where=account_sql & party_sql & line_query, - group_by=[table.id, line.party])) - if backend.name == 'sqlite': - sqlite_apply_types(query, types) - cursor.execute(*query) - for row in cursor: - id_ = pair(*row[0:2]) - for i, name in enumerate(names, 2): - result[name][id_] = row[i] + account_sql = fields.SQL_OPERATORS['in'](table.id, account_ids) + party_sql = fields.SQL_OPERATORS['in'](line.party, party_ids) + query = (table.join(line, 'LEFT', + condition=line.account == table.id + ).select(*columns, + where=account_sql & party_sql & line_query, + group_by=[table.id, line.party])) + if backend.name == 'sqlite': + sqlite_apply_types(query, types) + cursor.execute(*query) + for row in cursor: + id_ = pair(*row[0:2]) + for i, name in enumerate(names, 2): + result[name][id_] = row[i] for record in c_records: for name in names: if name == 'line_count': @@ -1753,26 +1738,25 @@ class AccountDeferral(ModelSQL, ModelView): account_child = Account.__table__() balances = defaultdict(Decimal) - for sub_deferrals in grouped_slice(deferrals): - red_sql = fields.SQL_OPERATORS['in']( - table.id, [d.id for d in sub_deferrals]) - query = (table - .join(account, condition=table.account == account.id) - .join(account_child, - condition=(account_child.left >= account.left) - & (account_child.right <= account.right)) - .join(table_child, - condition=(table_child.account == account_child.id) - & (table_child.fiscalyear == table.fiscalyear)) - .select( - table.id.as_('id'), - Sum(table_child.debit - table_child.credit).as_('balance'), - where=red_sql, - group_by=table.id)) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - balances.update(dict(cursor)) + where = fields.SQL_OPERATORS['in']( + table.id, [d.id for d in deferrals]) + query = (table + .join(account, condition=table.account == account.id) + .join(account_child, + condition=(account_child.left >= account.left) + & (account_child.right <= account.right)) + .join(table_child, + condition=(table_child.account == account_child.id) + & (table_child.fiscalyear == table.fiscalyear)) + .select( + table.id.as_('id'), + Sum(table_child.debit - table_child.credit).as_('balance'), + where=where, + group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + balances.update(dict(cursor)) return balances def get_currency(self, name): @@ -2490,25 +2474,22 @@ class GeneralLedgerLine(DescriptionOriginMixin, ModelSQL, ModelView): party_ids.add(r.party.id) query = account_party.select( - account_party.account, account_party.party, account_party.id) - for sub_account_ids in grouped_slice(account_ids): - account_where = fields.SQL_OPERATORS['in']( - account_party.account, sub_account_ids) - for sub_party_ids in grouped_slice(party_ids): - query.where = (account_where - & fields.SQL_OPERATORS['in']( - account_party.party, sub_party_ids)) - cursor.execute(*query) - for account, party, id_ in cursor: - key = (account, party) - try: - account_party_ids = account_party2ids[key] - except KeyError: - # There can be more combinations of account-party in - # the database than from records - continue - for record_id in account_party_ids: - account_parties[record_id] = id_ + account_party.account, account_party.party, account_party.id, + where=fields.SQL_OPERATORS['in']( + account_party.account, account_ids) + & fields.SQL_OPERATORS['in']( + account_party.party, party_ids)) + cursor.execute(*query) + for account, party, id_ in cursor: + key = (account, party) + try: + account_party_ids = account_party2ids[key] + except KeyError: + # There can be more combinations of account-party in + # the database than from records + continue + for record_id in account_party_ids: + account_parties[record_id] = id_ return account_parties @classmethod diff --git a/modules/account/company.py b/modules/account/company.py index 8eaad4a99e..e387db76c8 100644 --- a/modules/account/company.py +++ b/modules/account/company.py @@ -12,7 +12,7 @@ from trytond.model import fields from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction @@ -65,26 +65,24 @@ class Company(metaclass=PoolMeta): else: code = name today_where = Literal(True) - for sub_companies in grouped_slice(companies): - sub_ids = [p.id for p in sub_companies] - company_where = fields.SQL_OPERATORS['in']( - account.company, sub_ids) - query = (line - .join(account, - condition=account.id == line.account) - .join(account_type, - condition=account.type == account_type.id) - .select(account.company.as_('id'), amount.as_(name), - where=(getattr(account_type, code) - & (line.reconciliation == Null) - & company_where - & today_where), - group_by=account.company)) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - for company_id, value in cursor: - amounts[name][company_id] = value.quantize(exp[company_id]) + company_where = fields.SQL_OPERATORS['in']( + account.company, map(int, companies)) + query = (line + .join(account, + condition=account.id == line.account) + .join(account_type, + condition=account.type == account_type.id) + .select(account.company.as_('id'), amount.as_(name), + where=(getattr(account_type, code) + & (line.reconciliation == Null) + & company_where + & today_where), + group_by=account.company)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + for company_id, value in cursor: + amounts[name][company_id] = value.quantize(exp[company_id]) return amounts @classmethod diff --git a/modules/account/journal.py b/modules/account/journal.py index 29c388f71f..5aa3f03a8a 100644 --- a/modules/account/journal.py +++ b/modules/account/journal.py @@ -13,8 +13,7 @@ from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Eval -from trytond.tools import ( - grouped_slice, is_full_text, lstrip_wildcard, sqlite_apply_types) +from trytond.tools import is_full_text, lstrip_wildcard, sqlite_apply_types from trytond.transaction import Transaction STATES = { @@ -111,29 +110,27 @@ class Journal( & (move.date <= context.get('end_date')) & ~Coalesce(account_type.receivable, False) & ~Coalesce(account_type.payable, False) - & (move.company == company.id)) - for sub_journals in grouped_slice(journals): - sub_journals = list(sub_journals) - red_sql = fields.SQL_OPERATORS['in']( - move.journal, [j.id for j in sub_journals]) - query = line.join(move, 'LEFT', condition=line.move == move.id - ).join(account, 'LEFT', condition=line.account == account.id - ).join(account_type, 'LEFT', - condition=account.type == account_type.id - ).select( - move.journal, - Sum(line.debit).as_('debit'), - Sum(line.credit).as_('credit'), - where=where & red_sql, - group_by=move.journal) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC', 'NUMERIC']) - cursor.execute(*query) - for journal_id, debit, credit in cursor: - result['debit'][journal_id] = company.currency.round(debit) - result['credit'][journal_id] = company.currency.round(credit) - result['balance'][journal_id] = company.currency.round( - debit - credit) + & (move.company == company.id) + & fields.SQL_OPERATORS['in']( + move.journal, [j.id for j in journals])) + query = line.join(move, 'LEFT', condition=line.move == move.id + ).join(account, 'LEFT', condition=line.account == account.id + ).join(account_type, 'LEFT', + condition=account.type == account_type.id + ).select( + move.journal, + Sum(line.debit).as_('debit'), + Sum(line.credit).as_('credit'), + where=where, + group_by=move.journal) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC', 'NUMERIC']) + cursor.execute(*query) + for journal_id, debit, credit in cursor: + result['debit'][journal_id] = company.currency.round(debit) + result['credit'][journal_id] = company.currency.round(credit) + result['balance'][journal_id] = company.currency.round( + debit - credit) return result @classmethod @@ -154,16 +151,15 @@ class Journal( super().check_modification( mode, journals, values=values, external=external) if mode == 'write' and 'type' in values: - for sub_journals in grouped_slice(journals): - moves = Move.search([ - ('journal', 'in', [j.id for j in sub_journals]), - ('state', '=', 'posted') - ], order=[], limit=1) - if moves: - move, = moves - raise AccessError(gettext( - 'account.msg_journal_account_moves', - journal=move.journal.rec_name)) + moves = Move.search([ + ('journal', 'in', [j.id for j in journals]), + ('state', '=', 'posted') + ], order=[], limit=1) + if moves: + move, = moves + raise AccessError(gettext( + 'account.msg_journal_account_moves', + journal=move.journal.rec_name)) class JournalCashContext(ModelView): diff --git a/modules/account/move.py b/modules/account/move.py index c88dac1b31..d75ec99ee8 100644 --- a/modules/account/move.py +++ b/modules/account/move.py @@ -21,7 +21,7 @@ from trytond.pool import Pool from trytond.pyson import Bool, Eval, If, PYSONEncoder from trytond.report import Report from trytond.rpc import RPC -from trytond.tools import firstline, grouped_slice, sqlite_apply_types +from trytond.tools import firstline, sqlite_apply_types from trytond.transaction import Transaction, check_access from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -361,33 +361,32 @@ class Move(DescriptionOriginMixin, ModelSQL, ModelView): for company, moves in groupby(moves, key=lambda m: m.company): currency = company.currency - for sub_moves in grouped_slice(list(moves)): - red_sql = fields.SQL_OPERATORS['in']( - line.move, [m.id for m in sub_moves]) + where = fields.SQL_OPERATORS['in']( + line.move, [m.id for m in moves]) - valid_move_query = line.select( - line.move, - where=red_sql, - group_by=line.move, - having=Abs(Round( - Sum(line.debit - line.credit), - currency.digits)) < abs(currency.rounding)) - cursor.execute(*line.update( - [line.state], - ['valid'], - where=line.move.in_(valid_move_query))) + valid_move_query = line.select( + line.move, + where=where, + group_by=line.move, + having=Abs(Round( + Sum(line.debit - line.credit), + currency.digits)) < abs(currency.rounding)) + cursor.execute(*line.update( + [line.state], + ['valid'], + where=line.move.in_(valid_move_query))) - draft_move_query = line.select( - line.move, - where=red_sql, - group_by=line.move, - having=Abs(Round( - Sum(line.debit - line.credit), - currency.digits)) >= abs(currency.rounding)) - cursor.execute(*line.update( - [line.state], - ['draft'], - where=line.move.in_(draft_move_query))) + draft_move_query = line.select( + line.move, + where=where, + group_by=line.move, + having=Abs(Round( + Sum(line.debit - line.credit), + currency.digits)) >= abs(currency.rounding)) + cursor.execute(*line.update( + [line.state], + ['draft'], + where=line.move.in_(draft_move_query))) Transaction().counter += 1 for cache in Transaction().cache.values(): @@ -460,53 +459,49 @@ class Move(DescriptionOriginMixin, ModelSQL, ModelView): today = Date.today() currency = company.currency c_moves = list(c_moves) - for sub_moves in grouped_slice(c_moves): - sub_moves_ids = [m.id for m in sub_moves] + moves_ids = [m.id for m in moves] - cursor.execute(*move.select( - move.id, - where=fields.SQL_OPERATORS['in']( - move.id, sub_moves_ids) - & ~Exists(line.select( - line.move, - where=line.move == move.id)))) - try: - move_id, = cursor.fetchone() - except TypeError: - pass - else: - raise PostError( - gettext('account.msg_post_empty_move', - move=cls(move_id).rec_name)) + cursor.execute(*move.select( + move.id, + where=fields.SQL_OPERATORS['in'](move.id, moves_ids) + & ~Exists(line.select( + line.move, + where=line.move == move.id)))) + try: + move_id, = cursor.fetchone() + except TypeError: + pass + else: + raise PostError( + gettext('account.msg_post_empty_move', + move=cls(move_id).rec_name)) - cursor.execute(*line.select( - line.move, - where=fields.SQL_OPERATORS['in']( - line.move, sub_moves_ids), - group_by=line.move, - having=Abs(Round( - Sum(line.debit - line.credit), - currency.digits)) >= abs(currency.rounding))) - try: - move_id, = cursor.fetchone() - except TypeError: - pass - else: - raise PostError( - gettext('account.msg_post_unbalanced_move', - move=cls(move_id).rec_name)) + cursor.execute(*line.select( + line.move, + where=fields.SQL_OPERATORS['in'](line.move, moves_ids), + group_by=line.move, + having=Abs(Round( + Sum(line.debit - line.credit), + currency.digits)) >= abs(currency.rounding))) + try: + move_id, = cursor.fetchone() + except TypeError: + pass + else: + raise PostError( + gettext('account.msg_post_unbalanced_move', + move=cls(move_id).rec_name)) - cursor.execute(*line.select( - line.id, - where=fields.SQL_OPERATORS['in']( - line.move, sub_moves_ids) - & (line.debit == Decimal(0)) - & (line.credit == Decimal(0)) - & (line.reconciliation == Null) - & ((line.amount_second_currency == Null) - | (line.amount_second_currency == Decimal(0))) - )) - to_reconcile.extend(l for l, in cursor) + cursor.execute(*line.select( + line.id, + where=fields.SQL_OPERATORS['in'](line.move, moves_ids) + & (line.debit == Decimal(0)) + & (line.credit == Decimal(0)) + & (line.reconciliation == Null) + & ((line.amount_second_currency == Null) + | (line.amount_second_currency == Decimal(0))) + )) + to_reconcile.extend(l for l, in cursor) missing_number = defaultdict(list) for move in c_moves: @@ -832,63 +827,62 @@ class MoveLineMixin: date = Coalesce(line.maturity_date, move.date) for company, lines in groupby(lines, lambda l: l.company): - for sub_lines in grouped_slice(lines): - sub_lines = list(sub_lines) - id2currency = { - l.id: l.second_currency or company.currency - for l in sub_lines} - where = account.company == company.id - where_type = Literal(False) - if context.get('receivable', True): - where_type |= account_type.receivable - if context.get('payable', True): - where_type |= account_type.payable - where &= where_type - if not context.get('reconciled'): - if hasattr(cls, 'reconciliation'): - where &= line.reconciliation == Null - elif hasattr(cls, 'reconciled'): - where &= ~line.reconciled + lines = list(lines) + id2currency = { + l.id: l.second_currency or company.currency + for l in lines} + where = account.company == company.id + where_type = Literal(False) + if context.get('receivable', True): + where_type |= account_type.receivable + if context.get('payable', True): + where_type |= account_type.payable + where &= where_type + if not context.get('reconciled'): + if hasattr(cls, 'reconciliation'): + where &= line.reconciliation == Null + elif hasattr(cls, 'reconciled'): + where &= ~line.reconciled - currency = Coalesce(line.second_currency, company.currency.id) - sign = Case( - (account_type.statement == 'income', -1), - else_=1) - balance = sign * Case( - (line.amount_second_currency != Null, - line.amount_second_currency), - else_=line.debit - line.credit) - balance = Sum(balance, - window=Window( - [line.party, currency], - order_by=[date.asc.nulls_first, line.id.desc])) + currency = Coalesce(line.second_currency, company.currency.id) + sign = Case( + (account_type.statement == 'income', -1), + else_=1) + balance = sign * Case( + (line.amount_second_currency != Null, + line.amount_second_currency), + else_=line.debit - line.credit) + balance = Sum(balance, + window=Window( + [line.party, currency], + order_by=[date.asc.nulls_first, line.id.desc])) - party_where = Literal(False) - parties = {l.party for l in sub_lines} - if None in parties: - party_where |= line.party == parties.discard(None) - party_where |= fields.SQL_OPERATORS['in']( - line.party, map(int, parties)) - where &= party_where + party_where = Literal(False) + parties = {l.party for l in lines} + if None in parties: + party_where |= line.party == parties.discard(None) + party_where |= fields.SQL_OPERATORS['in']( + line.party, map(int, parties)) + where &= party_where - query = (line - .join(move, condition=line.move == move.id) - .join(account, condition=line.account == account.id) - .join( - account_type, - condition=account.type == account_type.id) - .select( - line.id.as_('id'), balance.as_('balance'), - where=where)) - query = query.select( - query.id, query.balance.as_('balance'), - where=fields.SQL_OPERATORS['in']( - query.id, [l.id for l in sub_lines])) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - balances.update( - (i, id2currency[i].round(a)) for (i, a) in cursor) + query = (line + .join(move, condition=line.move == move.id) + .join(account, condition=line.account == account.id) + .join( + account_type, + condition=account.type == account_type.id) + .select( + line.id.as_('id'), balance.as_('balance'), + where=where)) + query = query.select( + query.id, query.balance.as_('balance'), + where=fields.SQL_OPERATORS['in']( + query.id, [l.id for l in lines])) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + balances.update( + (i, id2currency[i].round(a)) for (i, a) in cursor) return balances def get_rec_name(self, name): diff --git a/modules/account/party.py b/modules/account/party.py index 0b082fa4d6..3e744eee56 100644 --- a/modules/account/party.py +++ b/modules/account/party.py @@ -16,7 +16,7 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.party.exceptions import EraseError from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval, If -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.tools import timezone as tz from trytond.transaction import Transaction @@ -200,14 +200,13 @@ class Party(CompanyMultiValueMixin, metaclass=PoolMeta): company = user.company currency = company.currency - for sub_parties in grouped_slice(parties): - query = cls._receivable_payable_query(names, company, sub_parties) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None] + ['NUMERIC'] * 4) - cursor.execute(*query) - for party_id, *values in cursor: - for name, value in zip(names, values): - amounts[name][party_id] = currency.round(value) + query = cls._receivable_payable_query(names, company, parties) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None] + ['NUMERIC'] * 4) + cursor.execute(*query) + for party_id, *values in cursor: + for name, value in zip(names, values): + amounts[name][party_id] = currency.round(value) return amounts @classmethod diff --git a/modules/account/period.py b/modules/account/period.py index 9d327202f2..e607010bea 100644 --- a/modules/account/period.py +++ b/modules/account/period.py @@ -4,6 +4,7 @@ from sql import Null from sql.operators import Equal, NotEqual +from trytond import backend from trytond.cache import Cache from trytond.const import OPERATORS from trytond.i18n import gettext @@ -226,7 +227,8 @@ class Period(Workflow, ModelSQL, ModelView): return lang = Lang.get() - for sub_periods in grouped_slice(periods): + for sub_periods in grouped_slice( + periods, backend.MAX_QUERY_PARAMS // 3): domain = ['OR'] for period in sub_periods: domain.append([ diff --git a/modules/account_asset/asset.py b/modules/account_asset/asset.py index aa0e4b4c57..6ed0e52813 100644 --- a/modules/account_asset/asset.py +++ b/modules/account_asset/asset.py @@ -16,7 +16,7 @@ from trytond.modules.company import CompanyReport from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Bool, Eval, If -from trytond.tools import cached_property, grouped_slice +from trytond.tools import cached_property from trytond.transaction import Transaction, check_access, without_check_access from trytond.wizard import ( Button, StateReport, StateTransition, StateView, Wizard) @@ -559,13 +559,11 @@ class Asset(Workflow, ModelSQL, ModelView): cls.create_lines(assets) moves = [] - lines = [] - for asset_ids in grouped_slice(assets): - lines += Line.search([ - ('asset', 'in', list(asset_ids)), - ('date', '<=', date), - ('move', '=', None), - ]) + lines = Line.search([ + ('asset', 'in', assets), + ('date', '<=', date), + ('move', '=', None), + ]) for line in lines: moves.append(line.asset.get_move(line)) Move.save(moves) diff --git a/modules/account_budget/account.py b/modules/account_budget/account.py index 77858c0b0e..19b7936764 100644 --- a/modules/account_budget/account.py +++ b/modules/account_budget/account.py @@ -17,7 +17,7 @@ from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Bool, Eval, If from trytond.rpc import RPC -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -55,12 +55,11 @@ class AmountMixin: balances = defaultdict(Decimal) for keys, grouped_records in groupby( records, key=cls._get_amount_group_key): - for sub_records in grouped_slice(list(grouped_records)): - query = cls._get_amount_query(sub_records, dict(keys)) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - balances.update(cursor) + query = cls._get_amount_query(list(grouped_records), dict(keys)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + balances.update(cursor) total = {n: {} for n in names} for record in records: @@ -212,10 +211,9 @@ class BudgetLineMixin( if backend.name == 'sqlite': sqlite_apply_types(query, [None, 'NUMERIC']) - for sub_ids in grouped_slice(ids): - query.where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*query) - amounts.update(cursor) + query.where = fields.SQL_OPERATORS['in'](table.id, ids) + cursor.execute(*query) + amounts.update(cursor) for record in records: amount = amounts[record.id] @@ -734,18 +732,19 @@ class BudgetLinePeriod(AmountMixin, ModelSQL, ModelView): transaction = Transaction() cursor = transaction.connection.cursor() table = cls.__table__() - for sub_ids in grouped_slice({p.budget_line.id for p in periods}): - cursor.execute(*table.select( - table.budget_line, - where=fields.SQL_OPERATORS['in']( - table.budget_line, sub_ids), - group_by=table.budget_line, - having=Sum(table.ratio) > 1, - limit=1)) - try: - line_id, = cursor.fetchone() - except TypeError: - continue + line_ids = {p.budget_line.id for p in periods} + cursor.execute(*table.select( + table.budget_line, + where=fields.SQL_OPERATORS['in']( + table.budget_line, line_ids), + group_by=table.budget_line, + having=Sum(table.ratio) > 1, + limit=1)) + try: + line_id, = cursor.fetchone() + except TypeError: + pass + else: line = Line(line_id) raise BudgetValidationError( gettext('account_budget.msg_budget_line_period_ratio', diff --git a/modules/account_deposit/party.py b/modules/account_deposit/party.py index 2a69430f7b..ffc8c754fe 100644 --- a/modules/account_deposit/party.py +++ b/modules/account_deposit/party.py @@ -12,7 +12,7 @@ from trytond.model import fields from trytond.modules.currency.fields import Monetary from trytond.modules.party.exceptions import EraseError from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction @@ -45,26 +45,25 @@ class Party(metaclass=PoolMeta): line_clause, _ = MoveLine.query_get(line) - for sub_parties in grouped_slice(parties): - party_clause = fields.SQL_OPERATORS['in']( - line.party, [p.id for p in sub_parties]) - query = (line - .join(account, condition=account.id == line.account) - .join(account_type, condition=account.type == account_type.id) - .select(line.party, - # Use credit - debit to positive deposit amount - Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0) - ).as_('debit'), - where=account_type.deposit - & party_clause - & (line.reconciliation == Null) - & (account.company == user.company.id) - & line_clause, - group_by=line.party)) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - values.update((p, currency.round(a)) for p, a in cursor) + party_clause = fields.SQL_OPERATORS['in']( + line.party, [p.id for p in parties]) + query = (line + .join(account, condition=account.id == line.account) + .join(account_type, condition=account.type == account_type.id) + .select(line.party, + # Use credit - debit to positive deposit amount + Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0) + ).as_('debit'), + where=account_type.deposit + & party_clause + & (line.reconciliation == Null) + & (account.company == user.company.id) + & line_clause, + group_by=line.party)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + values.update((p, currency.round(a)) for p, a in cursor) return values @classmethod diff --git a/modules/account_dunning_letter/dunning.py b/modules/account_dunning_letter/dunning.py index fae4ebd35c..5ae15e39ba 100644 --- a/modules/account_dunning_letter/dunning.py +++ b/modules/account_dunning_letter/dunning.py @@ -1,12 +1,11 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from itertools import chain, groupby +from itertools import groupby from operator import attrgetter from trytond.model import fields from trytond.modules.company import CompanyReport from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice from trytond.transaction import Transaction from trytond.wizard import StateReport @@ -120,19 +119,16 @@ class Letter(CompanyReport, metaclass=PoolMeta): """ pool = Pool() Line = pool.get('account.move.line') - payments = [] - for sub_parties in grouped_slice(parties): - payments.append(Line.search([ - ('account.type.receivable', '=', True), - ['OR', - ('debit', '<', 0), - ('credit', '>', 0), - ], - ('party', 'in', [p.id for p in sub_parties]), - ('reconciliation', '=', None), - ], - order=[('party', 'ASC'), ('id', 'ASC')])) - payments = list(chain(*payments)) + payments = Line.search([ + ('account.type.receivable', '=', True), + ['OR', + ('debit', '<', 0), + ('credit', '>', 0), + ], + ('party', 'in', parties), + ('reconciliation', '=', None), + ], + order=[('party', 'ASC'), ('id', 'ASC')]) return dict((party, list(payments)) for party, payments in groupby(payments, attrgetter('party'))) diff --git a/modules/account_es_sii/account.py b/modules/account_es_sii/account.py index 4369058a6a..a77a6e3752 100644 --- a/modules/account_es_sii/account.py +++ b/modules/account_es_sii/account.py @@ -245,15 +245,14 @@ class Period(metaclass=PoolMeta): def check_es_sii_posted_invoices(cls, periods): pool = Pool() Invoice = pool.get('account.invoice') - for sub_ids in grouped_slice(list(map(int, periods))): - invoices = Invoice.search([ - ('move.period', 'in', sub_ids), - ], limit=1, order=[]) - if invoices: - invoice, = invoices - raise ESSIIPostedInvoicesError(gettext( - 'account_es_sii.msg_es_sii_posted_invoices', - period=invoice.move.period.rec_name)) + invoices = Invoice.search([ + ('move.period', 'in', periods), + ], limit=1, order=[]) + if invoices: + invoice, = invoices + raise ESSIIPostedInvoicesError(gettext( + 'account_es_sii.msg_es_sii_posted_invoices', + period=invoice.move.period.rec_name)) class Invoice(metaclass=PoolMeta): diff --git a/modules/account_invoice/account.py b/modules/account_invoice/account.py index 661f30e17a..8b235bfa5a 100644 --- a/modules/account_invoice/account.py +++ b/modules/account_invoice/account.py @@ -1,6 +1,6 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from collections import OrderedDict +from collections import OrderedDict, defaultdict from itertools import islice from trytond.i18n import gettext @@ -10,7 +10,6 @@ from trytond.modules.account.exceptions import ClosePeriodError from trytond.modules.company.model import CompanyValueMixin from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .exceptions import CancelInvoiceMoveWarning @@ -224,13 +223,11 @@ class MoveLine(metaclass=PoolMeta): pool = Pool() InvoicePaymentLine = pool.get('account.invoice-account.move.line') - ids = list(map(int, lines)) - result = dict.fromkeys(ids, None) - for sub_ids in grouped_slice(ids): - payment_lines = InvoicePaymentLine.search([ - ('line', 'in', list(sub_ids)), - ]) - result.update({p.line.id: p.invoice.id for p in payment_lines}) + result = defaultdict(lambda: None) + payment_lines = InvoicePaymentLine.search([ + ('line', 'in', lines), + ]) + result.update({p.line.id: p.invoice.id for p in payment_lines}) return result @classmethod @@ -269,13 +266,10 @@ def _invoices_to_process(reconciliations): move_ids.add(line.move.id) others.update(line.reconciliations_delegated) - invoices = set() - for sub_ids in grouped_slice(move_ids): - sub_ids = list(sub_ids) - invoices.update(Invoice.search(['OR', - ('move', 'in', sub_ids), - ('additional_moves', 'in', sub_ids), - ])) + invoices = set(Invoice.search(['OR', + ('move', 'in', move_ids), + ('additional_moves', 'in', move_ids), + ])) if others: invoices.update(_invoices_to_process(Reconciliation.browse(others))) diff --git a/modules/account_invoice/invoice.py b/modules/account_invoice/invoice.py index 31169292ca..9736cb5984 100644 --- a/modules/account_invoice/invoice.py +++ b/modules/account_invoice/invoice.py @@ -828,17 +828,16 @@ class Invoice( type_name = cls.tax_amount._field.sql_type().base tax = InvoiceTax.__table__() - for sub_invoices in grouped_slice(invoices_no_cache): - red_sql = fields.SQL_OPERATORS['in']( - tax.invoice, map(int, sub_invoices)) - query = (tax.select(tax.invoice, - Coalesce(Sum(tax.amount), 0).as_(type_name), - where=red_sql, - group_by=tax.invoice)) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - tax_amount.update(cursor) + where = fields.SQL_OPERATORS['in']( + tax.invoice, map(int, invoices_no_cache)) + query = (tax.select(tax.invoice, + Coalesce(Sum(tax.amount), 0).as_(type_name), + where=where, + group_by=tax.invoice)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + tax_amount.update(cursor) # Float amount must be rounded to get the right precision if backend.name == 'sqlite': for invoice in invoices: @@ -915,14 +914,13 @@ class Invoice( def get_lines_to_pay(cls, invoices, name): cursor = Transaction().connection.cursor() lines = defaultdict(list) - for sub_invoices in grouped_slice(invoices): - query = cls._query_lines_to_pay(sub_invoices) - query = query.select( - query.invoice, query.line, - order_by=query.maturity_date.nulls_last) - cursor.execute(*query) - for invoice_id, line_id in cursor: - lines[invoice_id].append(line_id) + query = cls._query_lines_to_pay(invoices) + query = query.select( + query.invoice, query.line, + order_by=query.maturity_date.nulls_last) + cursor.execute(*query) + for invoice_id, line_id in cursor: + lines[invoice_id].append(line_id) return lines @classmethod @@ -934,27 +932,25 @@ class Invoice( move = Move.__table__() cursor = Transaction().connection.cursor() lines = defaultdict(list) - for sub_invoices in grouped_slice(invoices): - sub_invoices = list(sub_invoices) - lines_to_pay = cls._query_lines_to_pay(sub_invoices) - query = cls._query_lines_to_pay(sub_invoices) - query = ( - query - .join(line, - condition=query.reconciliation == line.reconciliation) - .join(move, condition=line.move == move.id) - .select( - query.invoice, line.id, - where=~Exists( - lines_to_pay.select( - lines_to_pay.line, - where=(lines_to_pay.invoice == query.invoice) - & (lines_to_pay.line == line.id))), - group_by=[query.invoice, line.id, move.date], - order_by=move.date.asc)) - cursor.execute(*query) - for invoice_id, line_id in cursor: - lines[invoice_id].append(line_id) + lines_to_pay = cls._query_lines_to_pay(invoices) + query = cls._query_lines_to_pay(invoices) + query = ( + query + .join(line, + condition=query.reconciliation == line.reconciliation) + .join(move, condition=line.move == move.id) + .select( + query.invoice, line.id, + where=~Exists( + lines_to_pay.select( + lines_to_pay.line, + where=(lines_to_pay.invoice == query.invoice) + & (lines_to_pay.line == line.id))), + group_by=[query.invoice, line.id, move.date], + order_by=move.date.asc)) + cursor.execute(*query) + for invoice_id, line_id in cursor: + lines[invoice_id].append(line_id) return lines @classmethod @@ -1146,16 +1142,13 @@ class Invoice( table = cls.__table__() cursor = Transaction().connection.cursor() - result = {} has_cache = ( (table.invoice_report_cache_id != Null) | (table.invoice_report_cache != Null)) - for sub_invoices in grouped_slice(invoices): - sub_ids = map(int, sub_invoices) - cursor.execute(*table.select(table.id, has_cache, - where=fields.SQL_OPERATORS['in'](table.id, sub_ids))) - result.update(cursor) - return result + cursor.execute(*table.select(table.id, has_cache, + where=fields.SQL_OPERATORS['in']( + table.id, map(int, invoices)))) + return dict(cursor) @property @fields.depends('lines', 'type') @@ -1896,13 +1889,11 @@ class Invoice( PaymentLine = pool.get('account.invoice-account.move.line') payments = defaultdict(list) - ids = list(map(int, lines)) - for sub_ids in grouped_slice(ids): - payment_lines = PaymentLine.search([ - ('line', 'in', list(sub_ids)), - ]) - for payment_line in payment_lines: - payments[payment_line.invoice].append(payment_line.line) + payment_lines = PaymentLine.search([ + ('line', 'in', lines), + ]) + for payment_line in payment_lines: + payments[payment_line.invoice].append(payment_line.line) to_write = [] for invoice, lines in payments.items(): @@ -2144,7 +2135,8 @@ class Invoice( def _check_similar(cls, invoices, type='in'): pool = Pool() Warning = pool.get('res.user.warning') - for sub_invoices in grouped_slice(invoices): + for sub_invoices in grouped_slice( + invoices, backend.MAX_QUERY_PARAMS // 10): sub_invoices = list(sub_invoices) domain = list(filter(None, (i._similar_domain() for i in sub_invoices diff --git a/modules/account_invoice_history/account.py b/modules/account_invoice_history/account.py index ad26dcd29c..630e46205f 100644 --- a/modules/account_invoice_history/account.py +++ b/modules/account_invoice_history/account.py @@ -1,5 +1,8 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. + +from collections import defaultdict + from sql import Null from sql.conditionals import Greatest from sql.functions import CurrentTimestamp @@ -7,7 +10,7 @@ from sql.functions import CurrentTimestamp from trytond import backend from trytond.model import ModelView, Workflow, fields from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction @@ -41,27 +44,25 @@ class Invoice(metaclass=PoolMeta): payment_term = PaymentTerm.__table__() cursor = Transaction().connection.cursor() - invoice_ids = [i.id for i in invoices] - datetimes = dict.fromkeys(invoice_ids) - for ids in grouped_slice(invoice_ids): - query = (table - .join(party, condition=table.party == party.id) - .join(address, condition=table.invoice_address == address.id) - .join(identifier, 'LEFT', - condition=table.party_tax_identifier == identifier.id) - .join(payment_term, 'LEFT', - condition=table.payment_term == payment_term.id) - .select(table.id, - Greatest(table.numbered_at, party.create_date, - address.create_date, identifier.create_date, - payment_term.create_date).as_('history_datetime'), - where=fields.SQL_OPERATORS['in'](table.id, ids) - & (table.numbered_at != Null) - & (table.state.in_(cls._history_states())))) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'DATETIME']) - cursor.execute(*query) - datetimes.update(cursor) + datetimes = defaultdict(lambda: None) + query = (table + .join(party, condition=table.party == party.id) + .join(address, condition=table.invoice_address == address.id) + .join(identifier, 'LEFT', + condition=table.party_tax_identifier == identifier.id) + .join(payment_term, 'LEFT', + condition=table.payment_term == payment_term.id) + .select(table.id, + Greatest(table.numbered_at, party.create_date, + address.create_date, identifier.create_date, + payment_term.create_date).as_('history_datetime'), + where=fields.SQL_OPERATORS['in'](table.id, map(int, invoices)) + & (table.numbered_at != Null) + & (table.state.in_(cls._history_states())))) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'DATETIME']) + cursor.execute(*query) + datetimes.update(cursor) return datetimes @classmethod diff --git a/modules/account_payment/account.py b/modules/account_payment/account.py index 5c5078ca0e..47fc864c0c 100644 --- a/modules/account_payment/account.py +++ b/modules/account_payment/account.py @@ -19,7 +19,6 @@ from trytond.modules.company.model import CompanyValueMixin from trytond.modules.currency.fields import Monetary from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import grouped_slice from trytond.transaction import Transaction, check_access, without_check_access from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -192,12 +191,9 @@ class MoveLine(metaclass=PoolMeta): where=table.account.in_(accounts))) if lines: - for sub_lines in grouped_slice(lines): - query.where = ( - table.account.in_(accounts) - & fields.SQL_OPERATORS['in']( - table.id, map(int, sub_lines))) - cursor.execute(*query) + query.where &= fields.SQL_OPERATORS['in']( + table.id, map(int, lines)) + cursor.execute(*query) else: cursor.execute(*query) diff --git a/modules/account_payment/payment.py b/modules/account_payment/payment.py index f3b32db5c2..2577397639 100644 --- a/modules/account_payment/payment.py +++ b/modules/account_payment/payment.py @@ -21,8 +21,7 @@ from trytond.modules.currency.fields import Monetary from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval, If from trytond.rpc import RPC -from trytond.tools import ( - cursor_dict, grouped_slice, sortable_values, sqlite_apply_types) +from trytond.tools import cursor_dict, sortable_values, sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import StateAction, Wizard @@ -201,32 +200,31 @@ class Group(ModelSQL, ModelView, ChatMixin): ).as_('payment_not_complete'), ] - for sub_groups in grouped_slice(groups): - query = payment.select(*columns, - where=fields.SQL_OPERATORS['in']( - payment.group, map(int, sub_groups)), - group_by=payment.group) - if backend.name == 'sqlite': - sqlite_apply_types( - query, [None, None, 'NUMERIC', 'NUMERIC', None]) - cursor.execute(*query) - for row in cursor_dict(cursor): - group = cls(row['group_id']) + query = payment.select(*columns, + where=fields.SQL_OPERATORS['in']( + payment.group, map(int, groups)), + group_by=payment.group) + if backend.name == 'sqlite': + sqlite_apply_types( + query, [None, None, 'NUMERIC', 'NUMERIC', None]) + cursor.execute(*query) + for row in cursor_dict(cursor): + group = cls(row['group_id']) - result['payment_count'][group.id] = row['payment_count'] - result['payment_complete'][group.id] = \ - not row['payment_not_complete'] + result['payment_count'][group.id] = row['payment_count'] + result['payment_complete'][group.id] = \ + not row['payment_not_complete'] - amount = row['payment_amount'] - succeeded = row['payment_amount_succeeded'] + amount = row['payment_amount'] + succeeded = row['payment_amount_succeeded'] - if amount is not None and backend.name == 'sqlite': - amount = group.company.currency.round(amount) - result['payment_amount'][group.id] = amount + if amount is not None and backend.name == 'sqlite': + amount = group.company.currency.round(amount) + result['payment_amount'][group.id] = amount - if succeeded is not None and backend.name == 'sqlite': - succeeded = group.company.currency.round(succeeded) - result['payment_amount_succeeded'][group.id] = succeeded + if succeeded is not None and backend.name == 'sqlite': + succeeded = group.company.currency.round(succeeded) + result['payment_amount_succeeded'][group.id] = succeeded for key in list(result.keys()): if key not in names: diff --git a/modules/account_payment_clearing/account.py b/modules/account_payment_clearing/account.py index 65d8b23230..6ef69aefba 100644 --- a/modules/account_payment_clearing/account.py +++ b/modules/account_payment_clearing/account.py @@ -1,7 +1,6 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice class Move(metaclass=PoolMeta): @@ -25,11 +24,9 @@ def _payments_to_update(reconciliations): moves.add(line.move) others.update(line.reconciliations_delegated) - payments = set() - for sub_moves in grouped_slice(moves): - payments.update(Payment.search([ - ('clearing_move', 'in', [m.id for m in sub_moves]), - ], order=[])) + payments = set(Payment.search([ + ('clearing_move', 'in', moves), + ], order=[])) if others: payments.update(_payments_to_update(Reconciliation.browse(others))) diff --git a/modules/account_payment_clearing/payment.py b/modules/account_payment_clearing/payment.py index d21c5323da..274bacfb81 100644 --- a/modules/account_payment_clearing/payment.py +++ b/modules/account_payment_clearing/payment.py @@ -13,7 +13,6 @@ from trytond.model import ModelView, Workflow, fields from trytond.modules.account.exceptions import AccountMissing from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, If, TimeDelta -from trytond.tools import grouped_slice from trytond.transaction import Transaction from trytond.wizard import Button, StateTransition, StateView, Wizard @@ -444,13 +443,12 @@ class Group(metaclass=PoolMeta): column = Min(column) else: column = BoolAnd(column) - for sub_groups in grouped_slice(groups): - cursor.execute(*payment.select( - payment.group, column, - where=fields.SQL_OPERATORS['in']( - payment.group, map(int, sub_groups)), - group_by=payment.group)) - result.update(cursor) + cursor.execute(*payment.select( + payment.group, column, + where=fields.SQL_OPERATORS['in']( + payment.group, map(int, groups)), + group_by=payment.group)) + result.update(cursor) return result @classmethod diff --git a/modules/account_payment_sepa/payment.py b/modules/account_payment_sepa/payment.py index dcaa0513a4..8365a117a2 100644 --- a/modules/account_payment_sepa/payment.py +++ b/modules/account_payment_sepa/payment.py @@ -4,6 +4,7 @@ import datetime import os import unicodedata import uuid +from collections import defaultdict from io import BytesIO from itertools import groupby from operator import attrgetter @@ -26,8 +27,7 @@ from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval, If from trytond.report import Report from trytond.tools import ( - cached_property, grouped_slice, is_full_text, lstrip_wildcard, - sortable_values) + cached_property, is_full_text, lstrip_wildcard, sortable_values) from trytond.transaction import Transaction from .sepa_handler import CAMT054 @@ -785,14 +785,12 @@ class Mandate(Workflow, ModelSQL, ModelView): payment = Payment.__table__() cursor = Transaction().connection.cursor() - has_payments = dict.fromkeys([m.id for m in mandates], False) - for sub_mandates in grouped_slice(mandates): - red_sql = fields.SQL_OPERATORS['in']( - payment.sepa_mandate, map(int, sub_mandates)) - cursor.execute(*payment.select(payment.sepa_mandate, Literal(True), - where=red_sql, - group_by=payment.sepa_mandate)) - has_payments.update(cursor) + has_payments = defaultdict(lambda: False) + cursor.execute(*payment.select(payment.sepa_mandate, Literal(True), + where=fields.SQL_OPERATORS['in']( + payment.sepa_mandate, map(int, mandates)), + group_by=payment.sepa_mandate)) + has_payments.update(cursor) return has_payments @@ -803,22 +801,20 @@ class Mandate(Workflow, ModelSQL, ModelView): payment = Payment.__table__() cursor = Transaction().connection.cursor() - is_first = dict.fromkeys([m.id for m in mandates], True) - for sub_mandates in grouped_slice(mandates): - red_sql = fields.SQL_OPERATORS['in']( - payment.sepa_mandate, map(int, sub_mandates)) - cursor.execute(*payment.select( - payment.sepa_mandate, Literal(False), - where=red_sql - & (payment.sepa_mandate_sequence_type != Null) - & ~( # Same as property rejected - (payment.state == 'failed') - & ((payment.sepa_return_reason_code != Null) - | (payment.sepa_return_reason_code != '')) - & (payment.sepa_return_reason_information - == '/RTYP/RJCT')), - group_by=payment.sepa_mandate)) - is_first.update(cursor) + is_first = defaultdict(lambda: True) + cursor.execute(*payment.select( + payment.sepa_mandate, Literal(False), + where=fields.SQL_OPERATORS['in']( + payment.sepa_mandate, map(int, mandates)) + & (payment.sepa_mandate_sequence_type != Null) + & ~( # Same as property rejected + (payment.state == 'failed') + & ((payment.sepa_return_reason_code != Null) + | (payment.sepa_return_reason_code != '')) + & (payment.sepa_return_reason_information + == '/RTYP/RJCT')), + group_by=payment.sepa_mandate)) + is_first.update(cursor) return is_first @classmethod diff --git a/modules/account_statement/bank.py b/modules/account_statement/bank.py index 1f6841eabe..bfb4643bc6 100644 --- a/modules/account_statement/bank.py +++ b/modules/account_statement/bank.py @@ -4,7 +4,6 @@ from trytond.i18n import gettext from trytond.modules.bank.exceptions import AccountValidationError from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice class Account(metaclass=PoolMeta): @@ -21,16 +20,14 @@ class Account(metaclass=PoolMeta): Journal = pool.get('account.statement.journal') if field_names and 'currency' not in field_names: return - for sub_accounts in grouped_slice(accounts): - sub_account_ids = [a.id for a in sub_accounts] - journals = Journal.search([ - ('bank_account', 'in', sub_account_ids), - ]) - for journal in journals: - if (journal.currency != journal.bank_account.currency - and journal.bank_account.currency): - raise AccountValidationError( - gettext('account_statement.msg_bank_account_currency', - bank_account=journal.bank_account.rec_name, - currency=journal.currency.rec_name, - journal=journal.rec_name)) + journals = Journal.search([ + ('bank_account', 'in', accounts), + ]) + for journal in journals: + if (journal.currency != journal.bank_account.currency + and journal.bank_account.currency): + raise AccountValidationError( + gettext('account_statement.msg_bank_account_currency', + bank_account=journal.bank_account.rec_name, + currency=journal.currency.rec_name, + journal=journal.rec_name)) diff --git a/modules/account_statement/journal.py b/modules/account_statement/journal.py index 8dec0a7cc7..fe7b0e9ade 100644 --- a/modules/account_statement/journal.py +++ b/modules/account_statement/journal.py @@ -1,6 +1,8 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. +from collections import defaultdict + from sql import Window from sql.functions import FirstValue @@ -10,7 +12,7 @@ from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Eval from trytond.rpc import RPC -from trytond.tools import cursor_dict, grouped_slice, sqlite_apply_types +from trytond.tools import cursor_dict, sqlite_apply_types from trytond.transaction import Transaction @@ -129,38 +131,37 @@ class Journal(DeactivableMixin, ModelSQL, ModelView): result = {} for name in names: - result[name] = dict.fromkeys(id2currency.keys(), None) + result[name] = defaultdict(lambda: None) - for sub_ids in grouped_slice(id2currency.keys()): - columns = [statement.journal] - w = Window( - partition=[statement.journal], - order_by=[statement.date.desc, statement.id.desc]) + columns = [statement.journal] + w = Window( + partition=[statement.journal], + order_by=[statement.date.desc, statement.id.desc]) + if 'last_amount' in names: + columns.append( + FirstValue(statement.end_balance, + window=w).as_('last_amount')) + if 'last_date' in names: + columns.append( + FirstValue(statement.date, window=w).as_('last_date')) + query = statement.select(*columns, + distinct=True, + where=(fields.SQL_OPERATORS['in']( + statement.journal, id2currency.keys()) + & (statement.state != 'cancelled'))) + if backend.name == 'sqlite': + types = [None] if 'last_amount' in names: - columns.append( - FirstValue(statement.end_balance, - window=w).as_('last_amount')) + types.append('NUMERIC') if 'last_date' in names: - columns.append( - FirstValue(statement.date, window=w).as_('last_date')) - query = statement.select(*columns, - distinct=True, - where=(fields.SQL_OPERATORS['in']( - statement.journal, sub_ids) - & (statement.state != 'cancelled'))) - if backend.name == 'sqlite': - types = [None] - if 'last_amount' in names: - types.append('NUMERIC') - if 'last_date' in names: - types.append('DATE') - sqlite_apply_types(query, types) - cursor.execute(*query) - for row in cursor_dict(cursor): - journal = row['journal'] - if 'last_amount' in names: - result['last_amount'][journal] = ( - id2currency[journal].round(row['last_amount'])) - if 'last_date' in names: - result['last_date'][journal] = row['last_date'] + types.append('DATE') + sqlite_apply_types(query, types) + cursor.execute(*query) + for row in cursor_dict(cursor): + journal = row['journal'] + if 'last_amount' in names: + result['last_amount'][journal] = ( + id2currency[journal].round(row['last_amount'])) + if 'last_date' in names: + result['last_date'][journal] = row['last_date'] return result diff --git a/modules/analytic_account/account.py b/modules/analytic_account/account.py index 29b7cbb8c8..ff82b58ca8 100644 --- a/modules/analytic_account/account.py +++ b/modules/analytic_account/account.py @@ -16,8 +16,7 @@ from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Eval, If, PYSONDecoder, PYSONEncoder -from trytond.tools import ( - grouped_slice, is_full_text, lstrip_wildcard, sqlite_apply_types) +from trytond.tools import is_full_text, lstrip_wildcard, sqlite_apply_types from trytond.transaction import Transaction from .exceptions import AccountValidationError @@ -140,16 +139,14 @@ class Account( return accounts = [ a for a in accounts if a.type in {'root', 'view', 'distribution'}] - for sub_accounts in grouped_slice(accounts): - sub_accounts = list(sub_accounts) - lines = Line.search([ - ('account', 'in', [a.id for a in sub_accounts]), - ], order=[], limit=1) - if lines: - line, = lines - raise AccountValidationError(gettext( - 'analytic_account.msg_account_wrong_type_line', - account=line.account.rec_name)) + lines = Line.search([ + ('account', 'in', accounts), + ], order=[], limit=1) + if lines: + line, = lines + raise AccountValidationError(gettext( + 'analytic_account.msg_account_wrong_type_line', + account=line.account.rec_name)) @fields.depends('company') def on_change_with_currency(self, name=None): @@ -313,17 +310,16 @@ class Account( mode, accounts, values=values, external=external) if mode == 'write' and 'root' in values: - for sub_records in grouped_slice(accounts): - entries = Entry.search([ - ('account', 'in', list(map(int, sub_records))), - ], - limit=1, order=[]) - if entries: - entry, = entries - raise AccessError(gettext( - 'analytic_account' - '.msg_analytic_account_root_change', - account=entry.account.rec_name)) + entries = Entry.search([ + ('account', 'in', accounts), + ], + limit=1, order=[]) + if entries: + entry, = entries + raise AccessError(gettext( + 'analytic_account' + '.msg_analytic_account_root_change', + account=entry.account.rec_name)) class AccountContext(ModelView): diff --git a/modules/commission/commission.py b/modules/commission/commission.py index f4956e3fff..7e24456561 100644 --- a/modules/commission/commission.py +++ b/modules/commission/commission.py @@ -1,6 +1,7 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. +from collections import defaultdict from decimal import Decimal from itertools import groupby @@ -21,7 +22,7 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import decistmt, grouped_slice, sqlite_apply_types +from trytond.tools import decistmt, sqlite_apply_types from trytond.transaction import Transaction, check_access from trytond.wizard import Button, StateAction, StateView, Wizard @@ -125,19 +126,17 @@ class Agent(DeactivableMixin, ModelSQL, ModelView): commission = Commission.__table__() cursor = Transaction().connection.cursor() - ids = [a.id for a in agents] - amounts = dict.fromkeys(ids, None) - for sub_ids in grouped_slice(ids): - where = fields.SQL_OPERATORS['in'](commission.agent, sub_ids) - where &= commission.invoice_line == Null - query = commission.select( - commission.agent, Sum(commission.amount).as_('pending_amount'), - where=where, - group_by=commission.agent) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - amounts.update(dict(cursor)) + amounts = defaultdict(lambda: None) + query = commission.select( + commission.agent, Sum(commission.amount).as_('pending_amount'), + where=fields.SQL_OPERATORS['in']( + commission.agent, map(int, agents)) + & (commission.invoice_line == Null), + group_by=commission.agent) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + amounts.update(dict(cursor)) if backend.name == 'sqlite': for agent_id, amount in amounts.items(): if amount is not None: diff --git a/modules/commission/invoice.py b/modules/commission/invoice.py index 79f83a7826..2dd35e55a1 100644 --- a/modules/commission/invoice.py +++ b/modules/commission/invoice.py @@ -7,7 +7,6 @@ from trytond.model import ModelView, Workflow, fields from trytond.modules.product import round_price from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -59,15 +58,13 @@ class Invoice(metaclass=PoolMeta): for company, c_invoices in groupby(invoices, key=lambda i: i.company): with Transaction().set_context(company=company.id): today = Date.today() - for sub_invoices in grouped_slice(list(c_invoices)): - ids = [i.id for i in sub_invoices] - for commission in Commission.search([ - ('date', '=', None), - ('origin.invoice', 'in', ids, - 'account.invoice.line'), - ]): - date = commission.origin.invoice.reconciled or today - date2commissions[date].append(commission) + for commission in Commission.search([ + ('date', '=', None), + ('origin.invoice', 'in', list(c_invoices), + 'account.invoice.line'), + ]): + date = commission.origin.invoice.reconciled or today + date2commissions[date].append(commission) to_write = [] for date, commissions in date2commissions.items(): to_write.append(commissions) @@ -103,19 +100,19 @@ class Invoice(metaclass=PoolMeta): to_delete = [] to_save = [] - for sub_invoices in grouped_slice(invoices_to_revert_commission): - ids = [i.id for i in sub_invoices] - to_delete += Commission.search([ - ('invoice_line', '=', None), - ('origin.invoice', 'in', ids, 'account.invoice.line'), - ]) - to_cancel = Commission.search([ - ('invoice_line', '!=', None), - ('origin.invoice', 'in', ids, 'account.invoice.line'), - ]) - for commission in Commission.copy(to_cancel): - commission.amount *= -1 - to_save.append(commission) + to_delete += Commission.search([ + ('invoice_line', '=', None), + ('origin.invoice', 'in', invoices_to_revert_commission, + 'account.invoice.line'), + ]) + to_cancel = Commission.search([ + ('invoice_line', '!=', None), + ('origin.invoice', 'in', invoices_to_revert_commission, + 'account.invoice.line'), + ]) + for commission in Commission.copy(to_cancel): + commission.amount *= -1 + to_save.append(commission) Commission.delete(to_delete) Commission.save(to_save) diff --git a/modules/customs/product.py b/modules/customs/product.py index 95a651fc35..86fecad994 100644 --- a/modules/customs/product.py +++ b/modules/customs/product.py @@ -3,7 +3,6 @@ from trytond.model import ModelSQL, ModelView, fields, sequence_ordered from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, Or -from trytond.tools import grouped_slice class Category(metaclass=PoolMeta): @@ -82,15 +81,10 @@ class Category(metaclass=PoolMeta): callback = super().on_delete(categories) - product_tariffcodes = set() - products = [str(t) for t in categories] - for products in grouped_slice(products): - product_tariffcodes.update(Product_TariffCode.search([ - 'product', 'in', list(products), - ])) + product_tariffcodes = Product_TariffCode.search([ + 'product', 'in', [str(t) for t in categories], + ]) if product_tariffcodes: - product_tariffcodes = Product_TariffCode.browse( - product_tariffcodes) callback.append( lambda: Product_TariffCode.delete(product_tariffcodes)) return callback @@ -157,15 +151,10 @@ class Template(metaclass=PoolMeta): callback = super().on_delete(templates) - product_tariffcodes = set() - products = [str(t) for t in templates] - for products in grouped_slice(products): - product_tariffcodes.update(Product_TariffCode.search([ - 'product', 'in', list(products), - ])) + product_tariffcodes = Product_TariffCode.search([ + 'product', 'in', [str(t) for t in templates], + ]) if product_tariffcodes: - product_tariffcodes = Product_TariffCode.browse( - product_tariffcodes) callback.append( lambda: Product_TariffCode.delete(product_tariffcodes)) return callback diff --git a/modules/marketing_automation/marketing_automation.py b/modules/marketing_automation/marketing_automation.py index d8a40eccbc..a8ae786bf9 100644 --- a/modules/marketing_automation/marketing_automation.py +++ b/modules/marketing_automation/marketing_automation.py @@ -26,7 +26,7 @@ from trytond.pool import Pool from trytond.pyson import Eval, If, PYSONDecoder, TimeDelta from trytond.report import Report, html_to_text, mjml_to_html from trytond.sendmail import SMTPDataManager, send_message_transactional -from trytond.tools import grouped_slice, pairwise_longest +from trytond.tools import pairwise_longest from trytond.tools.chart import sparkline from trytond.tools.email_ import format_address, has_rcpt, set_from_header from trytond.transaction import Transaction @@ -178,19 +178,18 @@ class Scenario( others.append(scenario) count = {name: defaultdict(int) for name in names} - for sub_others in grouped_slice(others): - cursor.execute(*record.select( - record.scenario, - Count(), - Count(filter_=record.blocked), - where=fields.SQL_OPERATORS['in']( - record.scenario, map(int, sub_others)), - group_by=record.scenario)) - for id_, all_, blocked in cursor: - if 'record_count' in count: - count['record_count'][id_] = all_ - if 'record_count_blocked' in count: - count['record_count_blocked'][id_] = blocked + cursor.execute(*record.select( + record.scenario, + Count(), + Count(filter_=record.blocked), + where=fields.SQL_OPERATORS['in']( + record.scenario, map(int, others)), + group_by=record.scenario)) + for id_, all_, blocked in cursor: + if 'record_count' in count: + count['record_count'][id_] = all_ + if 'record_count_blocked' in count: + count['record_count_blocked'][id_] = blocked for scenario in drafts: Model = pool.get(scenario.model) domain = PYSONDecoder({}).decode(scenario.domain) @@ -522,22 +521,21 @@ class Activity( cursor = Transaction().connection.cursor() count = {name: defaultdict(int) for name in names} - for sub_activities in grouped_slice(activities): - cursor.execute(*record_activity.select( - record_activity.activity, - Count(filter_=record_activity.state == 'done'), - Count(filter_=record_activity.email_opened), - Count(filter_=record_activity.email_clicked), - where=fields.SQL_OPERATORS['in']( - record_activity.activity, map(int, sub_activities)), - group_by=record_activity.activity)) - for id_, all_, email_opened, email_clicked in cursor: - if 'record_count' in count: - count['record_count'][id_] = all_ - if 'email_opened' in count: - count['email_opened'][id_] = email_opened - if 'email_clicked' in count: - count['email_clicked'][id_] = email_clicked + cursor.execute(*record_activity.select( + record_activity.activity, + Count(filter_=record_activity.state == 'done'), + Count(filter_=record_activity.email_opened), + Count(filter_=record_activity.email_clicked), + where=fields.SQL_OPERATORS['in']( + record_activity.activity, map(int, activities)), + group_by=record_activity.activity)) + for id_, all_, email_opened, email_clicked in cursor: + if 'record_count' in count: + count['record_count'][id_] = all_ + if 'email_opened' in count: + count['email_opened'][id_] = email_opened + if 'email_clicked' in count: + count['email_clicked'][id_] = email_clicked return count @classmethod diff --git a/modules/marketing_email/marketing.py b/modules/marketing_email/marketing.py index 210673d159..7b65f9ac92 100644 --- a/modules/marketing_email/marketing.py +++ b/modules/marketing_email/marketing.py @@ -22,7 +22,6 @@ from trytond.pool import Pool from trytond.pyson import Eval from trytond.report import Report, get_email, html_to_text, mjml_to_html from trytond.sendmail import SMTPDataManager, send_message_transactional -from trytond.tools import grouped_slice from trytond.tools.email_ import ( EmailNotValidError, format_address, normalize_email, set_from_header, validate_email) @@ -84,23 +83,22 @@ class Email(DeactivableMixin, ModelSQL, ModelView): result = {} web_user = 'web_user' in names if web_user: - web_users = dict.fromkeys(list(map(int, records))) + web_users = defaultdict(lambda: None) result['web_user'] = web_users party = 'party' in names if party: - parties = dict.fromkeys(list(map(int, records))) + parties = defaultdict(lambda: None) result['party'] = parties - for sub_records in grouped_slice(records): - email2id = {r.email: r.id for r in sub_records} - users = WebUser.search([ - ('email', 'in', list(email2id.keys())), - ]) - if web_user: - web_users.update((email2id[u.email], u.id) for u in users) - if party: - parties.update( - (email2id[u.email], u.party.id) - for u in users if u.party) + email2id = {r.email: r.id for r in records} + users = WebUser.search([ + ('email', 'in', list(email2id.keys())), + ]) + if web_user: + web_users.update((email2id[u.email], u.id) for u in users) + if party: + parties.update( + (email2id[u.email], u.party.id) + for u in users if u.party) return result @classmethod @@ -256,14 +254,14 @@ class EmailList(DeactivableMixin, ModelSQL, ModelView): subscribed = defaultdict(int) query = email.select( - email.list_, Count(), group_by=[email.list_]) - for sub_lists in grouped_slice(lists): - query.where = ( + email.list_, Count(), + where=( fields.SQL_OPERATORS['in']( - email.list_, map(int, sub_lists)) - & email.active) - cursor.execute(*query) - subscribed.update(cursor) + email.list_, map(int, lists)) + & email.active), + group_by=[email.list_]) + cursor.execute(*query) + subscribed.update(cursor) return subscribed def request_subscribe(self, email, from_=None): diff --git a/modules/product_price_list_cache/product.py b/modules/product_price_list_cache/product.py index 93c47c2523..83b79a5f61 100644 --- a/modules/product_price_list_cache/product.py +++ b/modules/product_price_list_cache/product.py @@ -10,7 +10,6 @@ from trytond.cache import Cache, freeze from trytond.model import ModelSQL, dualmethod, fields from trytond.pool import Pool, PoolMeta from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder -from trytond.tools import grouped_slice from trytond.transaction import Transaction dumps = partial( @@ -163,23 +162,17 @@ class PriceListCache(ModelSQL): if price_lists is None and products is None: cursor.execute(*cache.delete()) elif price_lists and products is None: - for sub_price_lists in grouped_slice(price_lists): - cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in']( - cache.price_list, [ - p.id for p in sub_price_lists]))) + cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in']( + cache.price_list, map(int, price_lists)))) elif price_lists is None and products: - for sub_products in grouped_slice(products): - cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in']( - cache.product, [p.id for p in sub_products]))) + cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in']( + cache.product, map(int, products)))) else: - for sub_products in grouped_slice(products): - for sub_price_lists in grouped_slice(price_lists): - cursor.execute(*cache.delete( - where=fields.SQL_OPERATORS['in']( - cache.price_list, [ - p.id for p in sub_price_lists]) - & fields.SQL_OPERATORS['in']( - cache.product, [p.id for p in sub_products]))) + cursor.execute(*cache.delete( + where=fields.SQL_OPERATORS['in']( + cache.price_list, map(int, price_lists)) + & fields.SQL_OPERATORS['in']( + cache.product, map(int, products)))) @classmethod def on_modification(cls, mode, records, field_names=None): diff --git a/modules/production_work/work.py b/modules/production_work/work.py index 0207d0d08b..2496d5529f 100644 --- a/modules/production_work/work.py +++ b/modules/production_work/work.py @@ -19,7 +19,7 @@ from trytond.modules.company.model import employee_field, set_employee from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, If, TimeDelta -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction from .exceptions import PickerError @@ -249,19 +249,16 @@ class Work(sequence_ordered(), ModelSQL, ModelView, ChatMixin): cursor = Transaction().connection.cursor() costs = defaultdict(Decimal) - for sub_works in grouped_slice(works): - red_sql = fields.SQL_OPERATORS['in']( - cycle.work, [w.id for w in sub_works]) - query = cycle.select( - cycle.work, Sum(Coalesce(cycle.cost, 0)).as_('cost'), - where=red_sql & (cycle.state == 'done'), - group_by=cycle.work) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - costs.update(cursor) - for cost in costs: - costs[cost] = round_price(costs[cost]) + query = cycle.select( + cycle.work, Sum(Coalesce(cycle.cost, 0)).as_('cost'), + where=fields.SQL_OPERATORS['in'](cycle.work, map(int, works)) + & (cycle.state == 'done'), + group_by=cycle.work) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + for work_id, cost in cursor: + costs[work_id] = round_price(cost) return costs @classmethod diff --git a/modules/project_invoice/invoice.py b/modules/project_invoice/invoice.py index 3a5bd5ae86..1f5f98e92e 100644 --- a/modules/project_invoice/invoice.py +++ b/modules/project_invoice/invoice.py @@ -10,7 +10,6 @@ from trytond.model import fields from trytond.modules.account_invoice.exceptions import ( InvoiceLineValidationError) from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -90,18 +89,16 @@ class InvoiceLine(metaclass=PoolMeta): durations = dict.fromkeys(map(int, lines)) query = ts_line.select( ts_line.invoice_line, Sum(ts_line.duration), + where=fields.SQL_OPERATORS['in']( + ts_line.invoice_line, map(int, lines)), group_by=ts_line.invoice_line) - for sub_lines in grouped_slice(lines): - query.where = fields.SQL_OPERATORS['in']( - ts_line.invoice_line, map(int, sub_lines)) - cursor.execute(*query) - - for line_id, duration in cursor: - # SQLite uses float for SUM - if (duration is not None - and not isinstance(duration, dt.timedelta)): - duration = dt.timedelta(seconds=duration) - durations[line_id] = duration + cursor.execute(*query) + for line_id, duration in cursor: + # SQLite uses float for SUM + if (duration is not None + and not isinstance(duration, dt.timedelta)): + duration = dt.timedelta(seconds=duration) + durations[line_id] = duration return durations @classmethod diff --git a/modules/project_invoice/project.py b/modules/project_invoice/project.py index 4491616a33..9dc9824080 100644 --- a/modules/project_invoice/project.py +++ b/modules/project_invoice/project.py @@ -18,7 +18,7 @@ from trytond.model.exceptions import AccessError from trytond.modules.currency.fields import Monetary from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import StateAction, Wizard @@ -112,28 +112,26 @@ class Progress: invoiced_progress = {} quantities = {} - for sub_works in grouped_slice(works): - sub_works = list(sub_works) - where = fields.SQL_OPERATORS['in']( - table.id, [x.id for x in sub_works if x.invoice_unit_price]) - cursor.execute(*table.join(progress, - condition=progress.work == table.id - ).select(table.id, Sum(progress.progress), - where=where, - group_by=table.id)) - invoiced_progress.update(dict(cursor)) + where = fields.SQL_OPERATORS['in']( + table.id, [x.id for x in works if x.invoice_unit_price]) + cursor.execute(*table.join(progress, + condition=progress.work == table.id + ).select(table.id, Sum(progress.progress), + where=where, + group_by=table.id)) + invoiced_progress.update(dict(cursor)) - for work in sub_works: - delta = ( - (work.progress or 0) - - invoiced_progress.get(work.id, 0.0)) - if work.invoice_unit_price and delta > 0: - quantity = delta - if work.price_list_hour: - quantity *= work.effort_hours - if work.unit_to_invoice: - quantity = work.unit_to_invoice.round(quantity) - quantities[work.id] = quantity + for work in works: + delta = ( + (work.progress or 0) + - invoiced_progress.get(work.id, 0.0)) + if work.invoice_unit_price and delta > 0: + quantity = delta + if work.price_list_hour: + quantity *= work.effort_hours + if work.unit_to_invoice: + quantity = work.unit_to_invoice.round(quantity) + quantities[work.id] = quantity return quantities @property @@ -161,32 +159,31 @@ class Progress: amounts = defaultdict(Decimal) work2currency = {} ids2work = dict((w.id, w) for w in works) - for sub_ids in grouped_slice(ids2work.keys()): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - query = (table.join(progress, - condition=progress.work == table.id - ).join(invoice_line, - condition=progress.invoice_line == invoice_line.id - ).select( - table.id, - Sum(Cast(progress.progress, 'NUMERIC') - * invoice_line.unit_price).as_('amount'), - where=where, - group_by=table.id)) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - for work_id, amount in cursor: - work = ids2work[work_id] - if work.price_list_hour: - amount *= Decimal(str(work.effort_hours)) - amounts[work_id] = amount + where = fields.SQL_OPERATORS['in'](table.id, list(ids2work.keys())) + query = (table.join(progress, + condition=progress.work == table.id + ).join(invoice_line, + condition=progress.invoice_line == invoice_line.id + ).select( + table.id, + Sum(Cast(progress.progress, 'NUMERIC') + * invoice_line.unit_price).as_('amount'), + where=where, + group_by=table.id)) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + for work_id, amount in cursor: + work = ids2work[work_id] + if work.price_list_hour: + amount *= Decimal(str(work.effort_hours)) + amounts[work_id] = amount - cursor.execute(*table.join(company, - condition=table.company == company.id - ).select(table.id, company.currency, - where=where)) - work2currency.update(cursor) + cursor.execute(*table.join(company, + condition=table.company == company.id + ).select(table.id, company.currency, + where=where)) + work2currency.update(cursor) currencies = Currency.browse(set(work2currency.values())) id2currency = {c.id: c for c in currencies} @@ -274,20 +271,19 @@ class Timesheet: line.work, Sum(line.duration), group_by=line.work) for upto, tworks in upto2tworks.items(): - for sub_ids in grouped_slice(tworks): - query.where = ( - fields.SQL_OPERATORS['in'](line.work, sub_ids) - & (line.invoice_line == Null)) - if upto: - query.where &= (line.date <= upto) - cursor.execute(*query) + query.where = ( + fields.SQL_OPERATORS['in'](line.work, tworks) + & (line.invoice_line == Null)) + if upto: + query.where &= (line.date <= upto) + cursor.execute(*query) - for twork_id, duration in cursor: - if duration: - # SQLite uses float for SUM - if not isinstance(duration, datetime.timedelta): - duration = datetime.timedelta(seconds=duration) - durations[twork2work[twork_id]] += duration + for twork_id, duration in cursor: + if duration: + # SQLite uses float for SUM + if not isinstance(duration, datetime.timedelta): + duration = datetime.timedelta(seconds=duration) + durations[twork2work[twork_id]] += duration quantities = {} for work in works: @@ -317,28 +313,26 @@ class Timesheet: amounts = {} work2currency = {} - work_ids = [w.id for w in works] - for sub_ids in grouped_slice(work_ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*table.join(timesheet_work, - condition=( - Concat(cls.__name__ + ',', table.id) - == timesheet_work.origin) - ).join(timesheet_line, - condition=timesheet_line.work == timesheet_work.id - ).join(invoice_line, - condition=timesheet_line.invoice_line == invoice_line.id - ).select(table.id, - Sum(timesheet_line.duration * invoice_line.unit_price), - where=where, - group_by=table.id)) - amounts.update(cursor) + where = fields.SQL_OPERATORS['in'](table.id, map(int, works)) + cursor.execute(*table.join(timesheet_work, + condition=( + Concat(cls.__name__ + ',', table.id) + == timesheet_work.origin) + ).join(timesheet_line, + condition=timesheet_line.work == timesheet_work.id + ).join(invoice_line, + condition=timesheet_line.invoice_line == invoice_line.id + ).select(table.id, + Sum(timesheet_line.duration * invoice_line.unit_price), + where=where, + group_by=table.id)) + amounts.update(cursor) - cursor.execute(*table.join(company, - condition=table.company == company.id - ).select(table.id, company.currency, - where=where)) - work2currency.update(cursor) + cursor.execute(*table.join(company, + condition=table.company == company.id + ).select(table.id, company.currency, + where=where)) + work2currency.update(cursor) currencies = Currency.browse(set(work2currency.values())) id2currency = {c.id: c for c in currencies} diff --git a/modules/project_revenue/work.py b/modules/project_revenue/work.py index 28cb08f4ef..55ca6a415e 100644 --- a/modules/project_revenue/work.py +++ b/modules/project_revenue/work.py @@ -13,7 +13,6 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.product import price_digits, round_price from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -93,23 +92,21 @@ class Work(metaclass=PoolMeta): work = Work.__table__() line = Line.__table__() - work_ids = [w.id for w in works] - for sub_ids in grouped_slice(work_ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*table.join(work, - condition=( - Concat(cls.__name__ + ',', table.id) == work.origin) - ).join(line, condition=line.work == work.id - ).select(table.id, Sum(line.cost_price * line.duration), - where=where, - group_by=[table.id])) - for work_id, cost in cursor: - # SQLite stores timedelta as float - if isinstance(cost, dt.timedelta): - cost = cost.total_seconds() - # Convert from seconds - cost /= 60 * 60 - yield work_id, Decimal(str(cost)) + where = fields.SQL_OPERATORS['in'](table.id, map(int, works)) + cursor.execute(*table.join(work, + condition=( + Concat(cls.__name__ + ',', table.id) == work.origin) + ).join(line, condition=line.work == work.id + ).select(table.id, Sum(line.cost_price * line.duration), + where=where, + group_by=[table.id])) + for work_id, cost in cursor: + # SQLite stores timedelta as float + if isinstance(cost, dt.timedelta): + cost = cost.total_seconds() + # Convert from seconds + cost /= 60 * 60 + yield work_id, Decimal(str(cost)) @classmethod def _purchase_cost(cls, works): @@ -131,27 +128,25 @@ class Work(metaclass=PoolMeta): invoice = Invoice.__table__() company = Company.__table__() - work_ids = [w.id for w in works] work2currency = {} iline2work = {} - for sub_ids in grouped_slice(work_ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*table.join(purchase_line, - condition=purchase_line.work == table.id - ).join(invoice_line, - condition=invoice_line.origin == Concat( - 'purchase.line,', purchase_line.id) - ).join(invoice, - condition=invoice_line.invoice == invoice.id - ).select(invoice_line.id, table.id, - where=where & ~invoice.state.in_(['draft', 'cancelled']))) - iline2work.update(cursor) + where = fields.SQL_OPERATORS['in'](table.id, map(int, works)) + cursor.execute(*table.join(purchase_line, + condition=purchase_line.work == table.id + ).join(invoice_line, + condition=invoice_line.origin == Concat( + 'purchase.line,', purchase_line.id) + ).join(invoice, + condition=invoice_line.invoice == invoice.id + ).select(invoice_line.id, table.id, + where=where & ~invoice.state.in_(['draft', 'cancelled']))) + iline2work.update(cursor) - cursor.execute(*table.join(company, - condition=table.company == company.id - ).select(table.id, company.currency, - where=where)) - work2currency.update(cursor) + cursor.execute(*table.join(company, + condition=table.company == company.id + ).select(table.id, company.currency, + where=where)) + work2currency.update(cursor) currencies = Currency.browse(set(work2currency.values())) id2currency = {c.id: c for c in currencies} diff --git a/modules/purchase/product.py b/modules/purchase/product.py index 07d240717c..fdae36a308 100644 --- a/modules/purchase/product.py +++ b/modules/purchase/product.py @@ -13,7 +13,7 @@ from trytond.modules.product import ( ProductDeactivatableMixin, price_digits, round_price) from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, If, TimeDelta -from trytond.tools import grouped_slice, is_full_text, lstrip_wildcard +from trytond.tools import is_full_text, lstrip_wildcard from trytond.transaction import Transaction from .exceptions import PurchaseUOMWarning @@ -154,16 +154,15 @@ class Product(metaclass=PoolMeta): supplier = context.get('supplier') if supplier: where &= purchase.party == supplier - for sub_products in grouped_slice(products): - query = (line - .join(purchase, condition=line.purchase == purchase.id) - .select( - Max(line.id), - where=where & fields.SQL_OPERATORS['in']( - line.product, map(int, sub_products)), - group_by=[line.product])) - cursor.execute(*query) - line_ids.extend(i for i, in cursor) + query = (line + .join(purchase, condition=line.purchase == purchase.id) + .select( + Max(line.id), + where=where & fields.SQL_OPERATORS['in']( + line.product, map(int, products)), + group_by=[line.product])) + cursor.execute(*query) + line_ids.extend(i for i, in cursor) lines = Line.browse(line_ids) uom = None diff --git a/modules/purchase_history/purchase.py b/modules/purchase_history/purchase.py index f4f0dbc7f7..4434058c77 100644 --- a/modules/purchase_history/purchase.py +++ b/modules/purchase_history/purchase.py @@ -4,7 +4,6 @@ from trytond.model import ModelView, Workflow, fields from trytond.pool import PoolMeta from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -45,12 +44,11 @@ class Purchase(metaclass=PoolMeta): table = cls.__table__() # Use SQL and before super to avoid two history entries - for sub_purchases in grouped_slice(purchases): - cursor.execute(*table.update( - [table.revision], - [table.revision + 1], - where=fields.SQL_OPERATORS['in']( - table.id, map(int, sub_purchases)))) + cursor.execute(*table.update( + [table.revision], + [table.revision + 1], + where=fields.SQL_OPERATORS['in']( + table.id, map(int, purchases)))) super().draft(purchases) diff --git a/modules/sale/sale_reporting.py b/modules/sale/sale_reporting.py index ace2085f00..ebe2daae35 100644 --- a/modules/sale/sale_reporting.py +++ b/modules/sale/sale_reporting.py @@ -15,7 +15,7 @@ from trytond.model import ModelSQL, ModelView, UnionMixin, fields, sum_tree from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Eval, If -from trytond.tools import grouped_slice, pairwise_longest +from trytond.tools import pairwise_longest from trytond.tools.chart import sparkline from trytond.transaction import Transaction from trytond.wizard import StateAction, StateTransition, Wizard @@ -513,16 +513,13 @@ class CustomerCategoryTree(ModelSQL, ModelView): categories = cls.search([ ('parent', 'child_of', [c.id for c in categories]), ]) - ids = [c.id for c in categories] reporting_customer_categories = [] - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - where = fields.SQL_OPERATORS['in']( - reporting_product_category.id, sub_ids) - cursor.execute( - *reporting_product_category.select( - reporting_product_category.id, where=where)) - reporting_customer_categories.extend(r for r, in cursor) + where = fields.SQL_OPERATORS['in']( + reporting_product_category.id, map(int, categories)) + cursor.execute( + *reporting_product_category.select( + reporting_product_category.id, where=where)) + reporting_customer_categories.extend(r for r, in cursor) result = {} reporting_product_categories = ReportingCustomerCategory.browse( @@ -755,16 +752,13 @@ class ProductCategoryTree(ModelSQL, ModelView): categories = cls.search([ ('parent', 'child_of', [c.id for c in categories]), ]) - ids = [c.id for c in categories] reporting_product_categories = [] - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - where = fields.SQL_OPERATORS['in']( - reporting_product_category.id, sub_ids) - cursor.execute( - *reporting_product_category.select( - reporting_product_category.id, where=where)) - reporting_product_categories.extend(r for r, in cursor) + where = fields.SQL_OPERATORS['in']( + reporting_product_category.id, map(int, categories)) + cursor.execute( + *reporting_product_category.select( + reporting_product_category.id, where=where)) + reporting_product_categories.extend(r for r, in cursor) result = {} reporting_product_categories = ReportingProductCategory.browse( diff --git a/modules/sale_complaint/complaint.py b/modules/sale_complaint/complaint.py index 21094680cb..ac7c301536 100644 --- a/modules/sale_complaint/complaint.py +++ b/modules/sale_complaint/complaint.py @@ -8,6 +8,7 @@ from decimal import Decimal from sql import Null from sql.functions import CharLength +from trytond import backend from trytond.i18n import gettext from trytond.model import ( ChatMixin, DeactivableMixin, Index, ModelSQL, ModelView, Workflow, fields) @@ -392,7 +393,8 @@ class Complaint(Workflow, ModelSQL, ModelView, ChatMixin): def _check_similar(cls, complaints): pool = Pool() Warning = pool.get('res.user.warning') - for sub_complaints in grouped_slice(complaints): + for sub_complaints in grouped_slice( + complaints, backend.MAX_QUERY_PARAMS // 10): sub_complaints = list(sub_complaints) domain = list(filter(None, (c._similar_domain() for c in sub_complaints))) diff --git a/modules/sale_credit_limit/party.py b/modules/sale_credit_limit/party.py index 589966252f..b0d3470414 100644 --- a/modules/sale_credit_limit/party.py +++ b/modules/sale_credit_limit/party.py @@ -4,7 +4,6 @@ from decimal import Decimal from itertools import groupby from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -22,43 +21,42 @@ class Party(metaclass=PoolMeta): company_id = Transaction().context.get('company') - for sub_parties in grouped_slice(parties): - id2party = {p.id: p for p in sub_parties} + id2party = {p.id: p for p in parties} - sales = Sale.search([ - ('company', '=', company_id), - ('party', 'in', list(id2party.keys())), - ('state', 'in', ['confirmed', 'processing']), - ], - order=[('party', None)]) - for party_id, sales in groupby(sales, lambda s: s.party.id): - party = id2party[party_id] - for sale in sales: - amount = 0 - for line in sale.lines: - quantity = line.credit_limit_quantity - if quantity is None: + sales = Sale.search([ + ('company', '=', company_id), + ('party', 'in', list(id2party.keys())), + ('state', 'in', ['confirmed', 'processing']), + ], + order=[('party', None)]) + for party_id, sales in groupby(sales, lambda s: s.party.id): + party = id2party[party_id] + for sale in sales: + amount = 0 + for line in sale.lines: + quantity = line.credit_limit_quantity + if quantity is None: + continue + for invoice_line in line.invoice_lines: + if invoice_line.type != 'line': continue - for invoice_line in line.invoice_lines: - if invoice_line.type != 'line': - continue - invoice = invoice_line.invoice - if invoice and invoice.move: - if line.unit: - quantity -= Uom.compute_qty( - invoice_line.unit or line.unit, - invoice_line.quantity, - line.unit, round=False) - else: - quantity -= invoice_line.quantity - if quantity > 0: - amount += Currency.compute( - sale.currency, - Decimal(str(quantity)) * line.unit_price, - party.currency, - round=False) - amounts[party.id] = party.currency.round( - amounts[sale.party.id] + amount) + invoice = invoice_line.invoice + if invoice and invoice.move: + if line.unit: + quantity -= Uom.compute_qty( + invoice_line.unit or line.unit, + invoice_line.quantity, + line.unit, round=False) + else: + quantity -= invoice_line.quantity + if quantity > 0: + amount += Currency.compute( + sale.currency, + Decimal(str(quantity)) * line.unit_price, + party.currency, + round=False) + amounts[party.id] = party.currency.round( + amounts[sale.party.id] + amount) return amounts @classmethod diff --git a/modules/sale_history/sale.py b/modules/sale_history/sale.py index dd7864757f..0dd7798ce0 100644 --- a/modules/sale_history/sale.py +++ b/modules/sale_history/sale.py @@ -4,7 +4,6 @@ from trytond.model import ModelView, Workflow, fields from trytond.pool import PoolMeta from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -41,12 +40,11 @@ class RevisionMixin: table = cls.__table__() # Use SQL and before super to avoid two history entries - for sub_records in grouped_slice(records): - cursor.execute(*table.update( - [table.revision], - [table.revision + 1], - where=fields.SQL_OPERATORS['in']( - table.id, map(int, sub_records)))) + cursor.execute(*table.update( + [table.revision], + [table.revision + 1], + where=fields.SQL_OPERATORS['in']( + table.id, map(int, records)))) super().draft(records) diff --git a/modules/sale_point/sale.py b/modules/sale_point/sale.py index 2c626014a3..c4d13a37b6 100644 --- a/modules/sale_point/sale.py +++ b/modules/sale_point/sale.py @@ -21,7 +21,7 @@ from trytond.modules.currency.fields import Monetary from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If -from trytond.tools import grouped_slice, sqlite_apply_types +from trytond.tools import sqlite_apply_types from trytond.transaction import Transaction from trytond.wizard import Button, StateTransition, StateView, Wizard @@ -82,15 +82,14 @@ class POS(ModelSQL, ModelView): if mode == 'write': if 'tax_included' in values: - for sub_points in grouped_slice(points): - if Sale.search([ - ('point', 'in', - list(map(int, sub_points))), - ], - limit=1, order=[]): - raise AccessError(gettext( - 'sale_point.' - 'msg_point_change_tax_included')) + if Sale.search([ + ('point', 'in', + list(map(int, points))), + ], + limit=1, order=[]): + raise AccessError(gettext( + 'sale_point.' + 'msg_point_change_tax_included')) class POSSale(Workflow, ModelSQL, ModelView, TaxableMixin, ChatMixin): @@ -859,29 +858,28 @@ class POSCashSession(Workflow, ModelSQL, ModelView): transfer = Transfer.__table__() cursor = Transaction().connection.cursor() balances = defaultdict(Decimal) - for sub_sessions in grouped_slice(sessions): - sub_ids = [s.id for s in sub_sessions] - query = Union( - payment.select( - payment.session.as_('session'), - Sum(payment.amount).as_('amount'), - where=fields.SQL_OPERATORS['in']( - payment.session, sub_ids), - group_by=[payment.session]), - transfer.select( - transfer.session.as_('session'), - Sum(transfer.amount).as_('amount'), - where=fields.SQL_OPERATORS['in']( - transfer.session, sub_ids), - group_by=[transfer.session]), - all_=True) - query = query.select( - query.session, Sum(query.amount).as_('amount'), - group_by=[query.session]) - if backend.name == 'sqlite': - sqlite_apply_types(query, [None, 'NUMERIC']) - cursor.execute(*query) - balances.update(cursor) + session_ids = [s.id for s in sessions] + query = Union( + payment.select( + payment.session.as_('session'), + Sum(payment.amount).as_('amount'), + where=fields.SQL_OPERATORS['in']( + payment.session, session_ids), + group_by=[payment.session]), + transfer.select( + transfer.session.as_('session'), + Sum(transfer.amount).as_('amount'), + where=fields.SQL_OPERATORS['in']( + transfer.session, session_ids), + group_by=[transfer.session]), + all_=True) + query = query.select( + query.session, Sum(query.amount).as_('amount'), + group_by=[query.session]) + if backend.name == 'sqlite': + sqlite_apply_types(query, [None, 'NUMERIC']) + cursor.execute(*query) + balances.update(cursor) for session in sessions: balances[session.id] = session.currency.round(balances[session.id]) return balances diff --git a/modules/sale_promotion_coupon/sale.py b/modules/sale_promotion_coupon/sale.py index 48cd4ff08c..3392dc43e0 100644 --- a/modules/sale_promotion_coupon/sale.py +++ b/modules/sale_promotion_coupon/sale.py @@ -15,7 +15,6 @@ from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval, If from trytond.sql.functions import DateRange from trytond.sql.operators import RangeOverlap -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .exceptions import PromotionCouponNumberDatesError @@ -299,14 +298,10 @@ class PromotionCouponNumber(DeactivableMixin, ModelSQL, ModelView): query, table, active = cls._active_query() query.columns = [table.id, active] - - result = {} - for sub_numbers in grouped_slice(numbers): - query.where = fields.SQL_OPERATORS['in']( - table.id, map(int, sub_numbers)) - cursor.execute(*query) - result.update(dict(cursor)) - return result + query.where = fields.SQL_OPERATORS['in']( + table.id, map(int, numbers)) + cursor.execute(*query) + return dict(cursor) @classmethod def search_active(cls, name, clause): diff --git a/modules/sale_stock_quantity/sale.py b/modules/sale_stock_quantity/sale.py index 0e8738c994..125a0771cd 100644 --- a/modules/sale_stock_quantity/sale.py +++ b/modules/sale_stock_quantity/sale.py @@ -9,7 +9,7 @@ from operator import attrgetter from trytond.i18n import gettext from trytond.model import ModelView, Workflow, fields from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice, sortable_values +from trytond.tools import sortable_values from trytond.transaction import Transaction from .exceptions import StockQuantityError, StockQuantityWarning @@ -104,12 +104,11 @@ class Sale(metaclass=PoolMeta): stock_date_start=date, stock_date_end=date): pbl = defaultdict(int) - for sub_products in grouped_slice(products): - sub_product_ids = [p.id for p in sub_products] - pbl.update(Product.products_by_location( - [warehouse.id], - with_childs=True, - grouping_filter=(sub_product_ids,))) + product_ids = [p.id for p in products] + pbl.update(Product.products_by_location( + [warehouse.id], + with_childs=True, + grouping_filter=(product_ids,))) delta = {} for key, qty in pbl.items(): _, product_id = key @@ -153,27 +152,26 @@ class Sale(metaclass=PoolMeta): (p, p.forecast_quantity) for p in products) # Remove quantities from other sales - for sub_products in grouped_slice(products): - other_lines = Line.search([ - ('sale.company', '=', self.company.id), - ('sale.state', 'in', self._stock_quantity_states()), - ('sale.id', '!=', self.id), - ('product', 'in', sub_products), - ('quantity', '>', 0), - ]) - for line in other_lines: - if line.warehouse != warehouse: - continue - if (line.shipping_date - and line.shipping_date > shipping_date): - continue - product = line.product - date = line.sale.sale_date or today - if date > today: - continue - quantity = Uom.compute_qty(line.unit, line.quantity, - product.default_uom, round=False) - quantities[line.warehouse][product] -= quantity + other_lines = Line.search([ + ('sale.company', '=', self.company.id), + ('sale.state', 'in', self._stock_quantity_states()), + ('sale.id', '!=', self.id), + ('product', 'in', products), + ('quantity', '>', 0), + ]) + for line in other_lines: + if line.warehouse != warehouse: + continue + if (line.shipping_date + and line.shipping_date > shipping_date): + continue + product = line.product + date = line.sale.sale_date or today + if date > today: + continue + quantity = Uom.compute_qty(line.unit, line.quantity, + product.default_uom, round=False) + quantities[line.warehouse][product] -= quantity for line in lines: warehouse = line.warehouse diff --git a/modules/sale_subscription_asset/stock.py b/modules/sale_subscription_asset/stock.py index b9405f0c13..957cd4d949 100644 --- a/modules/sale_subscription_asset/stock.py +++ b/modules/sale_subscription_asset/stock.py @@ -1,8 +1,10 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. + +from collections import defaultdict + from trytond.model import fields from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -24,20 +26,19 @@ class Lot(metaclass=PoolMeta): Date = pool.get('ir.date') SubscriptionLine = pool.get('sale.subscription.line') - subscribed_lines = {l.id: None for l in lots} + subscribed_lines = defaultdict(lambda: None) date = Transaction().context.get('date', Date.today()) - for sub_lots in grouped_slice(lots): - lines = SubscriptionLine.search([ - ('asset_lot', 'in', [l.id for l in sub_lots]), - [ - ('start_date', '<=', date), - ['OR', - ('end_date', '=', None), - ('end_date', '>', date), - ], - ] - ]) - subscribed_lines.update((s.asset_lot.id, s.id) for s in lines) + lines = SubscriptionLine.search([ + ('asset_lot', 'in', lots), + [ + ('start_date', '<=', date), + ['OR', + ('end_date', '=', None), + ('end_date', '>', date), + ], + ] + ]) + subscribed_lines.update((s.asset_lot.id, s.id) for s in lines) return subscribed_lines @classmethod diff --git a/modules/sale_supply_drop_shipment/product.py b/modules/sale_supply_drop_shipment/product.py index fbd774b8fa..9354c4e5e7 100644 --- a/modules/sale_supply_drop_shipment/product.py +++ b/modules/sale_supply_drop_shipment/product.py @@ -1,7 +1,6 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice class Product(metaclass=PoolMeta): @@ -13,17 +12,16 @@ class Product(metaclass=PoolMeta): Move = pool.get('stock.move') Shipment = pool.get('stock.shipment.drop') shipments = set() - for sub_products in grouped_slice(products): - domain = [ - ('unit_price_updated', '=', True), - cls._domain_moves_cost(), - ('product', 'in', [p.id for p in sub_products]), - ('shipment', 'like', 'stock.shipment.drop,%'), - ] - if start: - domain.append(('effective_date', '>=', start)) - for move in Move.search(domain, order=[]): - shipments.add(move.shipment) + domain = [ + ('unit_price_updated', '=', True), + cls._domain_moves_cost(), + ('product', 'in', products), + ('shipment', 'like', 'stock.shipment.drop,%'), + ] + if start: + domain.append(('effective_date', '>=', start)) + for move in Move.search(domain, order=[]): + shipments.add(move.shipment) shipments = Shipment.browse(list(shipments)) Shipment.set_cost(shipments) super().recompute_cost_price(products, start=start) diff --git a/modules/sale_supply_production/production.py b/modules/sale_supply_production/production.py index 8966526207..bc47e40cab 100644 --- a/modules/sale_supply_production/production.py +++ b/modules/sale_supply_production/production.py @@ -7,7 +7,6 @@ from trytond.i18n import gettext from trytond.model import Model, ModelView, Workflow from trytond.model.exceptions import AccessError from trytond.pool import Pool, PoolMeta -from trytond.tools import grouped_slice from trytond.transaction import Transaction, without_check_access @@ -21,11 +20,9 @@ def process_sale_supply(func): sales = set() with without_check_access(): - for sub_productions in grouped_slice(productions): - ids = [p.id for p in sub_productions] - sales.update([s.id for s in Sale.search([ - ('lines.productions', 'in', ids), - ])]) + sales.update([s.id for s in Sale.search([ + ('lines.productions', 'in', productions), + ])]) result = func(cls, productions) if sales: with transaction.set_context( diff --git a/modules/stock/inventory.py b/modules/stock/inventory.py index 4051a72b2d..ee2a450fec 100644 --- a/modules/stock/inventory.py +++ b/modules/stock/inventory.py @@ -11,7 +11,7 @@ from trytond.model import ( from trytond.model.exceptions import AccessError from trytond.pool import Pool from trytond.pyson import Bool, Eval, If -from trytond.tools import grouped_slice, is_full_text, lstrip_wildcard +from trytond.tools import is_full_text, lstrip_wildcard from trytond.transaction import Transaction from trytond.wizard import Button, StateTransition, StateView, Wizard @@ -257,11 +257,10 @@ class Inventory(Workflow, ModelSQL, ModelView, ChatMixin): else: product_ids = [l.product.id for l in inventory.lines] pbl = defaultdict(int) - for product_ids in grouped_slice(product_ids): - pbl.update(Product.products_by_location( - [inventory.location.id], - grouping=grouping, - grouping_filter=(list(product_ids),))) + pbl.update(Product.products_by_location( + [inventory.location.id], + grouping=grouping, + grouping_filter=(list(product_ids),))) # Update existing lines for line in inventory.lines: diff --git a/modules/stock/location.py b/modules/stock/location.py index 99cf8a9638..939f1dc3bd 100644 --- a/modules/stock/location.py +++ b/modules/stock/location.py @@ -14,7 +14,6 @@ from trytond.model import ( from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Eval, If, TimeDelta -from trytond.tools import grouped_slice from trytond.transaction import ( Transaction, inactive_records, without_check_access) @@ -314,18 +313,16 @@ class Location(DeactivableMixin, tree(), ModelSQL, ModelView): for (location_id, product), quantity in quantities.items(): if quantity and product not in consumables: empty.discard(location_id) - for sub_ids in grouped_slice(list(empty)): - sub_ids = list(sub_ids) - moves = Move.search([ - ('state', 'not in', ['done', 'cancelled']), - ['OR', - ('from_location', 'in', sub_ids), - ('to_location', 'in', sub_ids), - ], - ]) - for move in moves: - for location in [move.from_location, move.to_location]: - empty.discard(location.id) + moves = Move.search([ + ('state', 'not in', ['done', 'cancelled']), + ['OR', + ('from_location', 'in', empty), + ('to_location', 'in', empty), + ], + ]) + for move in moves: + for location in [move.from_location, move.to_location]: + empty.discard(location.id) return cls.browse(empty) @staticmethod @@ -434,15 +431,13 @@ class Location(DeactivableMixin, tree(), ModelSQL, ModelView): if not grouping: return {loc.id: None for loc in locations} - pbl = {} - for sub_locations in grouped_slice(locations): - location_ids = [l.id for l in sub_locations] - with Transaction().set_context(context): - pbl.update(Product.products_by_location( - location_ids, - grouping=grouping, - grouping_filter=grouping_filter, - with_childs=trans_context.get('with_childs', True))) + location_ids = [l.id for l in locations] + with Transaction().set_context(context): + pbl = Product.products_by_location( + location_ids, + grouping=grouping, + grouping_filter=grouping_filter, + with_childs=trans_context.get('with_childs', True)) return dict((loc.id, pbl.get((loc.id,) + key, 0)) for loc in locations) diff --git a/modules/stock/message.xml b/modules/stock/message.xml index 4c38169176..469144401d 100644 --- a/modules/stock/message.xml +++ b/modules/stock/message.xml @@ -13,9 +13,6 @@ this repository contains the full copyright notices and license terms. --> You cannot change the cost price of a product which is associated with stock moves. You must use the "Modify Cost Price" wizard. - - You cannot select more than %(max)s products to compute quantities. - Invalid cost price "%(cost_price)s" for product "%(product)s" with exception "%(exception)s". diff --git a/modules/stock/move.py b/modules/stock/move.py index 5781892d16..4e33df0141 100644 --- a/modules/stock/move.py +++ b/modules/stock/move.py @@ -19,7 +19,7 @@ from trytond.model.exceptions import AccessError from trytond.modules.product import price_digits, round_price from trytond.pool import Pool from trytond.pyson import Bool, Eval, Id, If, TimeDelta -from trytond.tools import cached_property, grouped_slice +from trytond.tools import cached_property from trytond.transaction import Transaction, without_check_access from .exceptions import MoveFutureWarning, MoveOriginWarning @@ -1314,7 +1314,6 @@ class Move(Workflow, ModelSQL, ModelView): connection = transaction.connection if database.has_select_for(): - count = database.IN_MAX // 2 PeriodCache = Period.get_cache(grouping) with connection.cursor() as cursor: for company_id in company_ids: @@ -1327,24 +1326,22 @@ class Move(Workflow, ModelSQL, ModelView): ], order=[('date', 'DESC')], limit=1) if periods: period, = periods - for sub_location_ids in grouped_slice(location_ids, count): - sub_location_ids = list(sub_location_ids) - table = cls.__table__() - query = table.select(Literal(1), - where=(fields.SQL_OPERATORS['in']( - table.to_location, sub_location_ids) - | fields.SQL_OPERATORS['in']( - table.from_location, sub_location_ids)) - & table.product.in_(product_ids) - & (table.company == company_id), - for_=For('UPDATE', nowait=True)) + table = cls.__table__() + query = table.select(Literal(1), + where=(fields.SQL_OPERATORS['in']( + table.to_location, location_ids) + | fields.SQL_OPERATORS['in']( + table.from_location, location_ids)) + & table.product.in_(product_ids) + & (table.company == company_id), + for_=For('UPDATE', nowait=True)) - if period: - query.where &= Coalesce( - table.effective_date, - table.planned_date, - datetime.date.max) > period.date - cursor.execute(*query) + if period: + query.where &= Coalesce( + table.effective_date, + table.planned_date, + datetime.date.max) > period.date + cursor.execute(*query) else: cls.lock() diff --git a/modules/stock/period.py b/modules/stock/period.py index 1ef8c6d3d0..8f6d2bbc06 100644 --- a/modules/stock/period.py +++ b/modules/stock/period.py @@ -1,6 +1,6 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from itertools import chain, groupby +from itertools import groupby from sql import For, Literal @@ -8,7 +8,6 @@ from trytond.i18n import gettext from trytond.model import Index, ModelSQL, ModelView, Workflow, fields from trytond.pool import Pool from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .exceptions import PeriodCloseError @@ -89,13 +88,10 @@ class Period(Workflow, ModelSQL, ModelView): def draft(cls, periods): for grouping in cls.groupings(): Cache = cls.get_cache(grouping) - caches = [] - for sub_periods in grouped_slice(periods): - caches.append(Cache.search([ - ('period', 'in', - [p.id for p in sub_periods]), - ], order=[])) - Cache.delete(list(chain(*caches))) + caches = Cache.search([ + ('period', 'in', periods), + ], order=[]) + Cache.delete(caches) @classmethod @ModelView.button diff --git a/modules/stock/product.py b/modules/stock/product.py index b83301f37c..65b3f7e565 100644 --- a/modules/stock/product.py +++ b/modules/stock/product.py @@ -17,7 +17,7 @@ from trytond.model.exceptions import AccessError from trytond.modules.product import price_digits, round_price from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, If, PYSONEncoder -from trytond.tools import decistmt, grouped_slice +from trytond.tools import decistmt from trytond.tools import timezone as tz from trytond.transaction import Transaction, without_check_access from trytond.wizard import ( @@ -35,14 +35,11 @@ def _find_moves(cls, records): field = 'product.template' else: field = 'product' - for sub_records in grouped_slice(records): - moves = Move.search([ - (field, 'in', list(map(int, sub_records))), - ], - limit=1, order=[]) - if moves: - return moves - return False + moves = Move.search([ + (field, 'in', list(map(int, records))), + ], + limit=1, order=[]) + return bool(moves) def _check_no_stock(cls, records): @@ -91,10 +88,9 @@ def _check_no_stock(cls, records): locations = Location.search([('type', '=', 'storage')]) location_ids = list(map(int, locations)) for company in Company.search([]): - for sub_records in grouped_slice(records): - record2locations = get_record_locations( - company, location_ids, sub_records, grouping) - raise_warning(cls, company, record2locations) + record2locations = get_record_locations( + company, location_ids, records, grouping) + raise_warning(cls, company, record2locations) class Template(metaclass=PoolMeta): @@ -279,12 +275,11 @@ class Product(StockMixin, object, metaclass=PoolMeta): pool = Pool() Move = pool.get('stock.move') deactivated = super().deactivate_replaced(products=products) - for sub_products in grouped_slice(deactivated): - moves = Move.search([ - ('product', 'in', list(map(int, sub_products))), - ('state', 'in', ['staging', 'draft']), - ]) - Move.delete(moves) + moves = Move.search([ + ('product', 'in', list(map(int, products))), + ('state', 'in', ['staging', 'draft']), + ]) + Move.delete(moves) return deactivated @classmethod @@ -367,16 +362,14 @@ class Product(StockMixin, object, metaclass=PoolMeta): cost = round_price(cost) costs[cost].append(product) - updated = [] - for sub_products in grouped_slice(products): - domain = [ - ('unit_price_updated', '=', True), - cls._domain_moves_cost(), - ('product', 'in', [p.id for p in sub_products]), - ] - if start: - domain.append(('effective_date', '>=', start)) - updated += Move.search(domain, order=[]) + domain = [ + ('unit_price_updated', '=', True), + cls._domain_moves_cost(), + ('product', 'in', products), + ] + if start: + domain.append(('effective_date', '>=', start)) + updated = Move.search(domain, order=[]) if updated: Move.write(updated, {'unit_price_updated': False}) @@ -675,10 +668,6 @@ class ProductQuantitiesByWarehouse(ModelSQL, ModelView): product = Product.__table__() from_ = move.join(product, condition=move.product == product.id) if product_template: - if len(product_template) > transaction.database.IN_MAX: - raise AccessError(gettext( - 'stock.msg_product_quantities_max', - max=transaction.database.IN_MAX)) product_clause = fields.SQL_OPERATORS['in']( product.template, product_template) else: @@ -692,10 +681,6 @@ class ProductQuantitiesByWarehouse(ModelSQL, ModelView): if isinstance(product, int): product = [product] if product: - if len(product) > transaction.database.IN_MAX: - raise AccessError(gettext( - 'stock.msg_product_quantities_max', - max=transaction.database.IN_MAX)) product_clause = fields.SQL_OPERATORS['in']( move.product, product) else: @@ -943,10 +928,6 @@ class ProductQuantitiesByWarehouseMove(ModelSQL, ModelView): product = Product.__table__() from_ = move.join(product, condition=move.product == product.id) if product_template: - if len(product_template) > transaction.database.IN_MAX: - raise AccessError(gettext( - 'stock.msg_product_quantities_max', - max=transaction.database.IN_MAX)) product_clause = fields.SQL_OPERATORS['in']( product.template, product_template) else: @@ -959,10 +940,6 @@ class ProductQuantitiesByWarehouseMove(ModelSQL, ModelView): if isinstance(product, int): product = [product] if product: - if len(product) > transaction.database.IN_MAX: - raise AccessError(gettext( - 'stock.msg_product_quantities_max', - max=transaction.database.IN_MAX)) product_clause = fields.SQL_OPERATORS['in']( move.product, product) else: @@ -1139,17 +1116,16 @@ class RecomputeCostPrice(Wizard): products = [] from_ = None - for sub_products in grouped_slice(products): - moves = Move.search([ - ('unit_price_updated', '=', True), - Product._domain_moves_cost(), - ('product', 'in', [p.id for p in sub_products]), - ], - order=[('effective_date', 'ASC')], - limit=1) - if moves: - move, = moves - from_ = min(from_ or datetime.date.max, move.effective_date) + moves = Move.search([ + ('unit_price_updated', '=', True), + Product._domain_moves_cost(), + ('product', 'in', products), + ], + order=[('effective_date', 'ASC')], + limit=1) + if moves: + move, = moves + from_ = min(from_ or datetime.date.max, move.effective_date) return {'from_': from_} def transition_recompute(self): diff --git a/modules/stock/stock_reporting_margin.py b/modules/stock/stock_reporting_margin.py index a587f406b6..60eb6fc564 100644 --- a/modules/stock/stock_reporting_margin.py +++ b/modules/stock/stock_reporting_margin.py @@ -15,7 +15,7 @@ from trytond.model import ModelSQL, ModelView, fields, sum_tree from trytond.modules.currency.fields import Monetary from trytond.pool import Pool from trytond.pyson import Eval, If -from trytond.tools import grouped_slice, pairwise_longest +from trytond.tools import pairwise_longest from trytond.tools.chart import sparkline from trytond.transaction import Transaction @@ -592,15 +592,11 @@ class CategoryTree(ModelSQL, ModelView): categories = cls.search([ ('parent', 'child_of', [c.id for c in categories]), ]) - ids = [c.id for c in categories] - reporting_categories = [] - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - where = fields.SQL_OPERATORS['in']( - reporting_category.id, sub_ids) - cursor.execute( - *reporting_category.select(reporting_category.id, where=where)) - reporting_categories.extend(r for r, in cursor) + cursor.execute(*reporting_category.select( + reporting_category.id, + where=fields.SQL_OPERATORS['in']( + reporting_category.id, map(int, categories)))) + reporting_categories = [r for r, in cursor] result = {} reporting_categories = ReportingCategory.browse(reporting_categories) diff --git a/modules/stock_forecast/forecast.py b/modules/stock_forecast/forecast.py index e5cb1d18c4..33bfeebf24 100644 --- a/modules/stock_forecast/forecast.py +++ b/modules/stock_forecast/forecast.py @@ -18,7 +18,6 @@ from trytond.pool import Pool from trytond.pyson import Bool, Eval, If from trytond.sql.functions import DateRange from trytond.sql.operators import RangeOverlap -from trytond.tools import grouped_slice from trytond.transaction import Transaction from trytond.wizard import Button, StateTransition, StateView, Wizard @@ -355,31 +354,30 @@ class ForecastLine(ModelSQL, ModelView): for forecast_id, lines in itertools.groupby(lines, key): forecast = Forecast(forecast_id) product2line = dict((line.product.id, line) for line in lines) - product_ids = product2line.keys() - for sub_ids in grouped_slice(product_ids): - where = fields.SQL_OPERATORS['in'](move.product, sub_ids) - cursor.execute(*move.join(location_from, - condition=move.from_location == location_from.id - ).join(location_to, - condition=move.to_location == location_to.id - ).select(move.product, Sum(move.internal_quantity), - where=where - & (location_from.left >= forecast.warehouse.left) - & (location_from.right <= forecast.warehouse.right) - & (location_to.left >= forecast.destination.left) - & (location_to.right <= forecast.destination.right) - & (move.state != 'cancelled') - & (Coalesce(move.effective_date, move.planned_date) - >= forecast.from_date) - & (Coalesce(move.effective_date, move.planned_date) - <= forecast.to_date) - & ((move.origin == Null) - | ~move.origin.like('stock.forecast.line,%')), - group_by=move.product)) - for product_id, quantity in cursor: - line = product2line[product_id] - result[line.id] = Uom.compute_qty( - line.product.default_uom, quantity, line.unit) + where = fields.SQL_OPERATORS['in']( + move.product, product2line.keys()) + cursor.execute(*move.join(location_from, + condition=move.from_location == location_from.id + ).join(location_to, + condition=move.to_location == location_to.id + ).select(move.product, Sum(move.internal_quantity), + where=where + & (location_from.left >= forecast.warehouse.left) + & (location_from.right <= forecast.warehouse.right) + & (location_to.left >= forecast.destination.left) + & (location_to.right <= forecast.destination.right) + & (move.state != 'cancelled') + & (Coalesce(move.effective_date, move.planned_date) + >= forecast.from_date) + & (Coalesce(move.effective_date, move.planned_date) + <= forecast.to_date) + & ((move.origin == Null) + | ~move.origin.like('stock.forecast.line,%')), + group_by=move.product)) + for product_id, quantity in cursor: + line = product2line[product_id] + result[line.id] = Uom.compute_qty( + line.product.default_uom, quantity, line.unit) return result @classmethod diff --git a/modules/stock_lot/stock.py b/modules/stock_lot/stock.py index cf951dd34d..f5f2b0268f 100644 --- a/modules/stock_lot/stock.py +++ b/modules/stock_lot/stock.py @@ -18,7 +18,6 @@ from trytond.modules.stock import StockMixin from trytond.modules.stock.exceptions import ShipmentCheckQuantityWarning from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, If, Len -from trytond.tools import grouped_slice from trytond.transaction import Transaction from trytond.wizard import Button, StateTransition, StateView, Wizard @@ -81,12 +80,10 @@ class Lot(DeactivableMixin, ModelSQL, ModelView, LotMixin, StockMixin): def get_quantity(cls, lots, name): location_ids = Transaction().context.get('locations') product_ids = list(set(l.product.id for l in lots)) - quantities = defaultdict(float) - for product_ids in grouped_slice(product_ids): - quantities.update(cls._get_quantity(lots, name, location_ids, - grouping=('product', 'lot',), - grouping_filter=(list(product_ids),))) - return quantities + return cls._get_quantity( + lots, name, location_ids, + grouping=('product', 'lot',), + grouping_filter=(product_ids,)) @classmethod def search_quantity(cls, name, domain=None): @@ -131,16 +128,13 @@ class Lot(DeactivableMixin, ModelSQL, ModelView, LotMixin, StockMixin): transaction = Transaction() def find_moves(cls, state=None): - for sub_records in grouped_slice(lots): - domain = [ - ('lot', 'in', [r.id for r in sub_records]) - ] - if state: - domain.append(('state', '=', state)) - moves = Move.search(domain, limit=1, order=[]) - if moves: - return True - return False + domain = [ + ('lot', 'in', lots) + ] + if state: + domain.append(('state', '=', state)) + moves = Move.search(domain, limit=1, order=[]) + return bool(moves) super().check_modification( mode, lots, values=values, external=external) diff --git a/modules/stock_lot_sled/stock.py b/modules/stock_lot_sled/stock.py index c2b107d124..cfe1db6bf6 100644 --- a/modules/stock_lot_sled/stock.py +++ b/modules/stock_lot_sled/stock.py @@ -11,7 +11,6 @@ from trytond.model.exceptions import AccessError from trytond.modules.stock.exceptions import PeriodCloseError from trytond.pool import Pool, PoolMeta from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .exceptions import LotExpiredError, LotExpiredWarning @@ -122,23 +121,21 @@ class Lot(LotSledMixin, metaclass=PoolMeta): if not periods: return period, = periods - for lots in grouped_slice(lots): - lot_ids = [l.id for l in lots] - moves = Move.search([ - ('lot', 'in', lot_ids), - ['OR', [ - ('effective_date', '=', None), - ('planned_date', '<=', period.date), - ], - ('effective_date', '<=', period.date), - ]], limit=1) - if moves: - move, = moves - raise AccessError( - gettext('stock_lot_sled' - '.msg_lot_modify_expiration_date_period_close', - lot=move.lot.rec_name, - move=move.rec_name)) + moves = Move.search([ + ('lot', 'in', lots), + ['OR', [ + ('effective_date', '=', None), + ('planned_date', '<=', period.date), + ], + ('effective_date', '<=', period.date), + ]], limit=1) + if moves: + move, = moves + raise AccessError( + gettext('stock_lot_sled' + '.msg_lot_modify_expiration_date_period_close', + lot=move.lot.rec_name, + move=move.rec_name)) class MoveAddLotsStartLot(LotSledMixin, metaclass=PoolMeta): diff --git a/modules/stock_shipment_measurements/stock.py b/modules/stock_shipment_measurements/stock.py index e924e19f13..ec41b10331 100644 --- a/modules/stock_shipment_measurements/stock.py +++ b/modules/stock_shipment_measurements/stock.py @@ -13,7 +13,6 @@ from trytond.model import Index, ModelSQL, ModelView, Workflow, fields from trytond.modules.company.model import CompanyValueMixin from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, Id -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -282,15 +281,9 @@ class MeasurementsMixin(object): where=table.state.in_(states))) if shipments is not None: - where = query.where - for sub_shipments in grouped_slice(shipments): - query.where = ( - where - & fields.SQL_OPERATORS['in']( - table.id, map(int, sub_shipments))) - cursor.execute(*query) - else: - cursor.execute(*query) + query.where &= fields.SQL_OPERATORS['in']( + table.id, map(int, shipments)) + cursor.execute(*query) if shipments: cls.write(shipments, {}) @@ -328,17 +321,15 @@ class MeasurementsMixin(object): weights, volumes = {}, {} states = cls._measurements_states() - for sub_shipments in grouped_slice( - (s for s in shipments + query.where = fields.SQL_OPERATORS['in']( + table.id, map(int, (s for s in shipments if s.state not in states or s.internal_weight is None - or s.internal_volume is None)): - query.where = fields.SQL_OPERATORS['in']( - table.id, [s.id for s in sub_shipments]) - cursor.execute(*query) - for id_, weight, volume in cursor: - weights[id_] = weight - volumes[id_] = volume + or s.internal_volume is None))) + cursor.execute(*query) + for id_, weight, volume in cursor: + weights[id_] = weight + volumes[id_] = volume for shipment in shipments: if 'weight' in names: diff --git a/modules/stock_supply/purchase_request.py b/modules/stock_supply/purchase_request.py index 3bf710c881..90b6a8aec7 100644 --- a/modules/stock_supply/purchase_request.py +++ b/modules/stock_supply/purchase_request.py @@ -7,7 +7,6 @@ from collections import defaultdict from trytond.model import ModelSQL, ValueMixin, fields from trytond.pool import Pool, PoolMeta from trytond.pyson import TimeDelta -from trytond.tools import grouped_slice from trytond.transaction import Transaction supply_period = fields.TimeDelta( @@ -80,60 +79,59 @@ class PurchaseRequest(metaclass=PoolMeta): # compute requests new_requests = [] - for (min_date, max_date), dates_products in date2products.items(): - for sub_products in grouped_slice(dates_products): - sub_products = Product.browse(sub_products) + for (min_date, max_date), sub_products in date2products.items(): + sub_products = Product.browse(sub_products) + + product2ops = {} + product2ops_other = {} + for product in sub_products: + for order_point in product.order_points: + if (order_point.company != company + or not order_point.warehouse_location): + continue + if order_point.type == 'purchase': + dict_ = product2ops + else: + dict_ = product2ops_other + dict_[ + (order_point.warehouse_location.id, + order_point.product.id) + ] = order_point + + product_ids = [p.id for p in sub_products] + with Transaction().set_context( + forecast=True, + stock_date_end=min_date): + pbl = Product.products_by_location(warehouse_ids, + with_childs=True, grouping_filter=(product_ids,)) + for warehouse_id in warehouse_ids: + min_date_qties = defaultdict(int, + ((x, pbl.pop((warehouse_id, x), 0)) + for x in product_ids)) + # Do not compute shortage for product + # with different order point + product_ids = [ + p.id for p in sub_products + if (warehouse_id, p.id) not in product2ops_other] + # Search for shortage between min-max + shortages = cls.get_shortage( + warehouse_id, product_ids, min_date, max_date, + min_date_qties=min_date_qties, + order_points=product2ops) - product2ops = {} - product2ops_other = {} for product in sub_products: - for order_point in product.order_points: - if (order_point.company != company - or not order_point.warehouse_location): - continue - if order_point.type == 'purchase': - dict_ = product2ops - else: - dict_ = product2ops_other - dict_[ - (order_point.warehouse_location.id, - order_point.product.id) - ] = order_point - - product_ids = [p.id for p in sub_products] - with Transaction().set_context( - forecast=True, - stock_date_end=min_date): - pbl = Product.products_by_location(warehouse_ids, - with_childs=True, grouping_filter=(product_ids,)) - for warehouse_id in warehouse_ids: - min_date_qties = defaultdict(int, - ((x, pbl.pop((warehouse_id, x), 0)) - for x in product_ids)) - # Do not compute shortage for product - # with different order point - product_ids = [ - p.id for p in sub_products - if (warehouse_id, p.id) not in product2ops_other] - # Search for shortage between min-max - shortages = cls.get_shortage( - warehouse_id, product_ids, min_date, max_date, - min_date_qties=min_date_qties, - order_points=product2ops) - - for product in sub_products: - if product.id not in shortages: - continue - shortage_date, product_quantity = shortages[product.id] - if shortage_date is None or product_quantity is None: - continue - order_point = product2ops.get( - (warehouse_id, product.id)) - # generate request values - request = cls.compute_request(product, - warehouse_id, shortage_date, product_quantity, - company, order_point) - new_requests.append(request) + if product.id not in shortages: + continue + shortage_date, product_quantity = shortages[product.id] + if shortage_date is None or product_quantity is None: + continue + order_point = product2ops.get( + (warehouse_id, product.id)) + # generate request values + request = cls.compute_request(product, + warehouse_id, shortage_date, product_quantity, + company, order_point) + new_requests.append(request) # delete purchase requests without a purchase line products = set(products) diff --git a/modules/stock_supply/shipment.py b/modules/stock_supply/shipment.py index ad36965cb5..d7e0fce8ca 100644 --- a/modules/stock_supply/shipment.py +++ b/modules/stock_supply/shipment.py @@ -5,7 +5,6 @@ from collections import defaultdict from trytond.model import ModelSQL, ModelView from trytond.pool import Pool -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -97,11 +96,9 @@ class ShipmentInternal(ModelSQL, ModelView): product_ids = [p.id for p in products] pbl = get_pbl(None, today, None) else: - product_ids = list(id2product.keys()) - product_ids.sort() + product_ids = sorted(id2product.keys()) pbl = defaultdict(int) - for sub_product_ids in grouped_slice(product_ids): - pbl.update(get_pbl(None, today, (list(sub_product_ids),))) + pbl.update(get_pbl(None, today, (product_ids,))) shipments = [] date = today @@ -188,10 +185,9 @@ class ShipmentInternal(ModelSQL, ModelView): for key, qty in get_pbl(date, date, None).items(): current_qties[key] += qty else: - for sub_product_ids in grouped_slice(product_ids): - for key, qty in get_pbl( - date, date, (list(sub_product_ids),)).items(): - current_qties[key] += qty + for key, qty in get_pbl( + date, date, (product_ids,)).items(): + current_qties[key] += qty if shipments: cls.save(shipments) diff --git a/modules/stock_supply_production/production.py b/modules/stock_supply_production/production.py index 6fc8822e67..5f226b3595 100644 --- a/modules/stock_supply_production/production.py +++ b/modules/stock_supply_production/production.py @@ -6,7 +6,6 @@ from collections import defaultdict from trytond.model import ModelSQL, ValueMixin, fields from trytond.pool import Pool, PoolMeta from trytond.pyson import TimeDelta -from trytond.tools import grouped_slice from trytond.transaction import Transaction supply_period = fields.TimeDelta( @@ -89,60 +88,59 @@ class Production(metaclass=PoolMeta): date2products[min_date, max_date].append(product) requests = [] - for (min_date, max_date), dates_products in date2products.items(): - for sub_products in grouped_slice(products): - sub_products = Product.browse(sub_products) + for (min_date, max_date), sub_products in date2products.items(): + sub_products = Product.browse(sub_products) + + product2ops = {} + product2ops_other = {} + for product in sub_products: + for order_point in product.order_points: + if (order_point.company != company + or not order_point.warehouse_location): + continue + if order_point.type == 'production': + dict_ = product2ops + else: + dict_ = product2ops_other + dict_[ + (order_point.warehouse_location.id, + order_point.product.id) + ] = order_point + + product_ids = [p.id for p in sub_products] + with Transaction().set_context( + forecast=True, + stock_date_end=min_date): + pbl = Product.products_by_location( + warehouse_ids, + with_childs=True, + grouping_filter=(product_ids,)) + + for warehouse in warehouses: + min_date_qties = defaultdict(int, + ((x, pbl.pop((warehouse.id, x), 0)) + for x in product_ids)) + # Do not compute shortage for product + # with different order point + product_ids = [ + p.id for p in sub_products + if (warehouse.id, p.id) not in product2ops_other] + # Search for shortage between min-max + shortages = cls.get_shortage( + warehouse.id, product_ids, min_date, max_date, + min_date_qties=min_date_qties, + order_points=product2ops) - product2ops = {} - product2ops_other = {} for product in sub_products: - for order_point in product.order_points: - if (order_point.company != company - or not order_point.warehouse_location): - continue - if order_point.type == 'production': - dict_ = product2ops - else: - dict_ = product2ops_other - dict_[ - (order_point.warehouse_location.id, - order_point.product.id) - ] = order_point - - product_ids = [p.id for p in sub_products] - with Transaction().set_context( - forecast=True, - stock_date_end=min_date): - pbl = Product.products_by_location( - warehouse_ids, - with_childs=True, - grouping_filter=(product_ids,)) - - for warehouse in warehouses: - min_date_qties = defaultdict(int, - ((x, pbl.pop((warehouse.id, x), 0)) - for x in product_ids)) - # Do not compute shortage for product - # with different order point - product_ids = [ - p.id for p in sub_products - if (warehouse.id, p.id) not in product2ops_other] - # Search for shortage between min-max - shortages = cls.get_shortage( - warehouse.id, product_ids, min_date, max_date, - min_date_qties=min_date_qties, - order_points=product2ops) - - for product in sub_products: - if product.id not in shortages: - continue - for date, quantity in shortages[product.id]: - order_point = product2ops.get( - (warehouse.id, product.id)) - req = cls.compute_request(product, warehouse, - quantity, date, company, order_point) - req.set_planned_start_date() - requests.append(req) + if product.id not in shortages: + continue + for date, quantity in shortages[product.id]: + order_point = product2ops.get( + (warehouse.id, product.id)) + req = cls.compute_request(product, warehouse, + quantity, date, company, order_point) + req.set_planned_start_date() + requests.append(req) cls.save(requests) cls.set_moves(requests) return requests diff --git a/modules/timesheet/work.py b/modules/timesheet/work.py index 1cd273e446..b0494cafc7 100644 --- a/modules/timesheet/work.py +++ b/modules/timesheet/work.py @@ -2,6 +2,7 @@ # this repository contains the full copyright notices and license terms. import datetime +from collections import defaultdict from sql import Literal from sql.aggregate import Sum @@ -10,7 +11,6 @@ from trytond.i18n import gettext from trytond.model import ModelSQL, ModelStorage, ModelView, Unique, fields from trytond.pool import Pool from trytond.pyson import Bool, Eval, If -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .exceptions import CompanyValidationError @@ -92,8 +92,7 @@ class Work(ModelSQL, ModelView): table_w = cls.__table__() line = Line.__table__() - ids = [w.id for w in works] - durations = dict.fromkeys(ids, None) + durations = defaultdict(lambda: None) where = Literal(True) if context.get('from_date'): where &= line.date >= context['from_date'] @@ -105,16 +104,15 @@ class Work(ModelSQL, ModelView): query_table = table_w.join(line, 'LEFT', condition=line.work == table_w.id) - for sub_ids in grouped_slice(ids): - cursor.execute(*query_table.select(table_w.id, Sum(line.duration), - where=where - & fields.SQL_OPERATORS['in'](table_w.id, sub_ids), - group_by=table_w.id)) - for work_id, duration in cursor: - # SQLite uses float for SUM - if duration and not isinstance(duration, datetime.timedelta): - duration = datetime.timedelta(seconds=duration) - durations[work_id] = duration + cursor.execute(*query_table.select(table_w.id, Sum(line.duration), + where=where + & fields.SQL_OPERATORS['in'](table_w.id, map(int, works)), + group_by=table_w.id)) + for work_id, duration in cursor: + # SQLite uses float for SUM + if duration and not isinstance(duration, datetime.timedelta): + duration = datetime.timedelta(seconds=duration) + durations[work_id] = duration return durations def get_work(self, name): diff --git a/modules/web_shop_shopify/product.py b/modules/web_shop_shopify/product.py index 9e5b9909d1..7aca43832e 100644 --- a/modules/web_shop_shopify/product.py +++ b/modules/web_shop_shopify/product.py @@ -13,7 +13,7 @@ from trytond.model import Exclude, ModelSQL, ModelView, fields from trytond.modules.product.exceptions import TemplateValidationError from trytond.pool import Pool, PoolMeta from trytond.pyson import Bool, Eval, If -from trytond.tools import grouped_slice, slugify +from trytond.tools import slugify from trytond.transaction import Transaction from . import graphql @@ -481,11 +481,9 @@ class Product_TariffCode(IdentifiersUpdateMixin, metaclass=PoolMeta): elif isinstance(record.product, Category): categories.add(record.product) if categories: - for sub_categories in grouped_slice(list(categories)): - templates.update(Template.search([ - ('customs_category', 'in', - [c.id for c in sub_categories]), - ])) + templates.update(Template.search([ + ('customs_category', 'in', map(int, categories)), + ])) templates = Template.browse(list(templates)) return Template.get_shopify_identifier_to_update(templates) @@ -579,11 +577,9 @@ class AttributeSet(IdentifiersUpdateMixin, metaclass=PoolMeta): def get_shopify_identifier_to_update(cls, sets): pool = Pool() Template = pool.get('product.template') - templates = [] - for sub_sets in grouped_slice(sets): - templates.extend(Template.search([ - ('attribute_set', 'in', [s.id for s in sub_sets]), - ])) + templates = Template.search([ + ('attribute_set', 'in', sets), + ]) return Template.get_shopify_identifier_to_update(templates) diff --git a/modules/web_shortener/web.py b/modules/web_shortener/web.py index 0e26cb7c8f..71f2405644 100644 --- a/modules/web_shortener/web.py +++ b/modules/web_shortener/web.py @@ -3,6 +3,7 @@ import logging import string +from collections import defaultdict from urllib.parse import quote, urljoin from sql.aggregate import Count @@ -10,7 +11,6 @@ from sql.aggregate import Count import trytond.config as config from trytond.model import ModelSQL, ModelView, fields from trytond.pool import Pool -from trytond.tools import grouped_slice from trytond.transaction import Transaction from trytond.url import http_host from trytond.wsgi import Base64Converter @@ -52,13 +52,13 @@ class ShortenedURL(ModelSQL, ModelView): access = URLAccess.__table__() cursor = Transaction().connection.cursor() - counts = {s.id: 0 for s in shortened_urls} - for sub_ids in grouped_slice(shortened_urls): - cursor.execute(*access.select( - access.url, Count(access.id), - where=fields.SQL_OPERATORS['in'](access.url, sub_ids), - group_by=[access.url])) - counts.update(cursor) + counts = defaultdict(int) + cursor.execute(*access.select( + access.url, Count(access.id), + where=fields.SQL_OPERATORS['in']( + access.url, map(int, shortened_urls)), + group_by=[access.url])) + counts.update(cursor) return counts @classmethod diff --git a/trytond-gis/trytond_gis/postgis/__init__.py b/trytond-gis/trytond_gis/postgis/__init__.py index be62f08573..94c339cd48 100644 --- a/trytond-gis/trytond_gis/postgis/__init__.py +++ b/trytond-gis/trytond_gis/postgis/__init__.py @@ -1,17 +1,18 @@ # This file is part of trytond_gis. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. -from trytond.backend.postgresql.database import ( - DatabaseDataError, DatabaseIntegrityError, DatabaseOperationalError, - DatabaseTimeoutError) +from trytond.backend.postgresql import ( + MAX_QUERY_PARAMS, DatabaseDataError, DatabaseIntegrityError, + DatabaseOperationalError, DatabaseTimeoutError) from .database import Database from .table import TableHandler __all__ = [ Database, - DatabaseIntegrityError, DatabaseDataError, + DatabaseIntegrityError, DatabaseOperationalError, DatabaseTimeoutError, + MAX_QUERY_PARAMS, TableHandler, ] diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG index b53127ab52..286770b9dc 100644 --- a/trytond/CHANGELOG +++ b/trytond/CHANGELOG @@ -1,3 +1,5 @@ +* Deprecate ``grouped_slice`` without size +* Replace ``Database.IN_MAX`` by ``backend.MAX_QUERY_PARAMS`` * Deprecate ``reduce_ids`` for ``SQL_OPERATORS['in']`` * Use array for ``in`` operators * Update to Psycopg 3 diff --git a/trytond/doc/ref/backend.rst b/trytond/doc/ref/backend.rst index 01e42665b9..f1856e96ca 100644 --- a/trytond/doc/ref/backend.rst +++ b/trytond/doc/ref/backend.rst @@ -16,6 +16,10 @@ The backend manages the database connection and schema manipulation. The name of the backend configured. +.. attribute:: MAX_QUERY_PARAMS + + The maximum number of parameters supported by the backend. + Database ======== diff --git a/trytond/trytond/admin.py b/trytond/trytond/admin.py index b352af2834..a1bb68f17a 100644 --- a/trytond/trytond/admin.py +++ b/trytond/trytond/admin.py @@ -217,14 +217,13 @@ def validate(models, percentage=100): if not models: models = sorted([n for n, _ in pool.iterobject()]) ratio = min(100, percentage) / 100 - in_max = Transaction().database.IN_MAX + limit = 1_000 for name in models: logger.info("validate: %s", name) Model = pool.get(name) if not issubclass(Model, ModelStorage): continue offset = 0 - limit = in_max while True: records = Model.search( [], order=[('id', 'ASC')], offset=offset, limit=limit) diff --git a/trytond/trytond/backend/__init__.py b/trytond/trytond/backend/__init__.py index 3db79b28e2..5b7200fad2 100644 --- a/trytond/trytond/backend/__init__.py +++ b/trytond/trytond/backend/__init__.py @@ -13,7 +13,7 @@ import trytond.config as config __all__ = [ 'name', 'Database', 'TableHandler', 'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError', - 'DatabaseTimeoutError'] + 'DatabaseTimeoutError', 'MAX_QUERY_PARAMS'] name = urllib.parse.urlparse(config.get('database', 'uri', default='')).scheme @@ -37,3 +37,4 @@ DatabaseDataError = _module.DatabaseDataError DatabaseOperationalError = _module.DatabaseOperationalError DatabaseTimeoutError = _module.DatabaseTimeoutError TableHandler = _module.TableHandler +MAX_QUERY_PARAMS = _module.MAX_QUERY_PARAMS diff --git a/trytond/trytond/backend/database.py b/trytond/trytond/backend/database.py index 3da718c277..81c7be925f 100644 --- a/trytond/trytond/backend/database.py +++ b/trytond/trytond/backend/database.py @@ -16,7 +16,6 @@ class DatabaseInterface(object): Define generic interface for database connection ''' flavor = None - IN_MAX = 1000 def __new__(cls, name=''): return object.__new__(cls) diff --git a/trytond/trytond/backend/postgresql/__init__.py b/trytond/trytond/backend/postgresql/__init__.py index 05a01ca5be..3739956543 100644 --- a/trytond/trytond/backend/postgresql/__init__.py +++ b/trytond/trytond/backend/postgresql/__init__.py @@ -6,7 +6,14 @@ from .database import ( DatabaseOperationalError, DatabaseTimeoutError) from .table import TableHandler +MAX_QUERY_PARAMS = 50_000 # rounded down from 65_535 + __all__ = [ - Database, TableHandler, - DatabaseIntegrityError, DatabaseDataError, DatabaseOperationalError, - DatabaseTimeoutError] + Database, + DatabaseDataError, + DatabaseIntegrityError, + DatabaseOperationalError, + DatabaseTimeoutError, + MAX_QUERY_PARAMS, + TableHandler, + ] diff --git a/trytond/trytond/backend/postgresql/database.py b/trytond/trytond/backend/postgresql/database.py index dd20341179..ce73dcdcad 100644 --- a/trytond/trytond/backend/postgresql/database.py +++ b/trytond/trytond/backend/postgresql/database.py @@ -29,7 +29,6 @@ from sql.operators import Any, BinaryOperator, Concat, NotEqual from trytond import __series__, config from trytond.backend.database import DatabaseInterface, SQLType from trytond.sql.operators import RangeOperator -from trytond.tools import grouped_slice from .table import index_method @@ -486,11 +485,11 @@ class Database(DatabaseInterface): def lock_records(cls, connection, table, ids): table = Table(table) cursor = connection.cursor() - for sub_ids in grouped_slice(ids): - where = table.id == Any(list(sub_ids)) - query = table.select( - Literal(1), where=where, for_=For('UPDATE', nowait=True)) - cursor.execute(*query) + query = table.select( + Literal(1), + where=table.id == Any(list(ids)), + for_=For('UPDATE', nowait=True)) + cursor.execute(*query) def lock_id(self, id, timeout=None): if not timeout: diff --git a/trytond/trytond/backend/sqlite/__init__.py b/trytond/trytond/backend/sqlite/__init__.py index ac1515602a..3df29910f2 100644 --- a/trytond/trytond/backend/sqlite/__init__.py +++ b/trytond/trytond/backend/sqlite/__init__.py @@ -9,10 +9,17 @@ from .database import ( DatabaseOperationalError, DatabaseTimeoutError) from .table import TableHandler +MAX_QUERY_PARAMS = 200 # estimation from the SQLITE_MAX_EXPR_DEPTH=1_000 + if sqlite3.sqlite_version_info < (3, 30, 0): sys.exit("Tryton sqlite backend requires version 3.30.0 or higher") __all__ = [ - Database, TableHandler, - DatabaseIntegrityError, DatabaseDataError, DatabaseOperationalError, - DatabaseTimeoutError] + Database, + DatabaseDataError, + DatabaseIntegrityError, + DatabaseOperationalError, + DatabaseTimeoutError, + MAX_QUERY_PARAMS, + TableHandler, + ] diff --git a/trytond/trytond/backend/sqlite/database.py b/trytond/trytond/backend/sqlite/database.py index 3e546de9ed..3e0f16e855 100644 --- a/trytond/trytond/backend/sqlite/database.py +++ b/trytond/trytond/backend/sqlite/database.py @@ -444,7 +444,6 @@ class Database(DatabaseInterface): flavor = Flavor( paramstyle='qmark', function_mapping=MAPPING, max_limit=-1, filter_=True) - IN_MAX = 200 TYPES_MAPPING = { 'BIGINT': SQLType('INTEGER', 'INTEGER'), diff --git a/trytond/trytond/ir/cron.py b/trytond/trytond/ir/cron.py index 8d29f98b1f..13dd67c162 100644 --- a/trytond/trytond/ir/cron.py +++ b/trytond/trytond/ir/cron.py @@ -22,7 +22,6 @@ from trytond.model import ( from trytond.pool import Pool from trytond.pyson import Eval from trytond.status import processing -from trytond.tools import grouped_slice from trytond.tools import timezone as tz from trytond.transaction import Transaction, TransactionError from trytond.worker import run_task @@ -139,16 +138,15 @@ class Cron(DeactivableMixin, ModelSQL, ModelView): with transaction.new_transaction() as transaction: cursor = transaction.connection.cursor() For = database.get_select_for_skip_locked() - for sub_crons in grouped_slice(crons): - ids = [c.id for c in sub_crons] - query = table.select( - table.id, - where=fields.SQL_OPERATORS['in'](table.id, ids), - for_=For('UPDATE')) - cursor.execute(*query) - not_running = {i for i, in cursor} - running.update( - (i, True) for i in ids if i not in not_running) + ids = [c.id for c in crons] + query = table.select( + table.id, + where=fields.SQL_OPERATORS['in'](table.id, ids), + for_=For('UPDATE')) + cursor.execute(*query) + not_running = {i for i, in cursor} + running.update( + (i, True) for i in ids if i not in not_running) return running @classmethod diff --git a/trytond/trytond/ir/message.py b/trytond/trytond/ir/message.py index f1b67caa61..944b003a86 100644 --- a/trytond/trytond/ir/message.py +++ b/trytond/trytond/ir/message.py @@ -6,7 +6,6 @@ from collections import defaultdict from trytond.cache import Cache from trytond.model import ModelSQL, ModelView, fields from trytond.pool import Pool -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .lang import get_parent_language as get_parent @@ -111,22 +110,20 @@ class Message(ModelSQL, ModelView): id2translations = {} other_translations = defaultdict(list) - for sub_messages in grouped_slice(messages): - res_ids = [m.id for m in sub_messages] - translations = Translation.search([ - ('lang', '=', INTERNAL_LANG), + translations = Translation.search([ + ('lang', '=', INTERNAL_LANG), + ('type', '=', 'model'), + ('name', '=', f'{cls.__name__},text'), + ('res_id', 'in', messages), + ]) + id2translations.update((t.res_id, t) for t in translations) + for translation in Translation.search([ + ('lang', '!=', INTERNAL_LANG), ('type', '=', 'model'), ('name', '=', f'{cls.__name__},text'), - ('res_id', 'in', res_ids), - ]) - id2translations.update((t.res_id, t) for t in translations) - for translation in Translation.search([ - ('lang', '!=', INTERNAL_LANG), - ('type', '=', 'model'), - ('name', '=', f'{cls.__name__},text'), - ('res_id', 'in', res_ids), - ]): - other_translations[translation.res_id].append(translation) + ('res_id', 'in', messages), + ]): + other_translations[translation.res_id].append(translation) to_save = [] for message in messages: diff --git a/trytond/trytond/ir/model.py b/trytond/trytond/ir/model.py index f579393180..97b6556b42 100644 --- a/trytond/trytond/ir/model.py +++ b/trytond/trytond/ir/model.py @@ -24,7 +24,7 @@ from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder from trytond.pyson import Bool, Eval, PYSONDecoder from trytond.report import Report from trytond.rpc import RPC -from trytond.tools import cursor_dict, grouped_slice, is_instance_method +from trytond.tools import cursor_dict, is_instance_method from trytond.tools.string_ import StringMatcher from trytond.transaction import Transaction, without_check_access from trytond.wizard import ( @@ -1215,14 +1215,14 @@ class ModelButtonClick(DeactivableMixin, ModelSQL, ModelView): } for r in records]) clicks = defaultdict(list) - for records in grouped_slice(records): - records = cls.search([ - ('button', '=', button.id), - ('record_id', 'in', [r.id for r in records]), - ], order=[('record_id', 'ASC')]) - clicks.update( - (k, list(v)) for k, v in groupby( - records, key=lambda c: c.record_id)) + records = cls.search([ + ('button', '=', button.id), + ('record_id', 'in', records), + ], + order=[('record_id', 'ASC')]) + clicks.update( + (k, list(v)) for k, v in groupby( + records, key=lambda c: c.record_id)) return clicks @classmethod @@ -1230,13 +1230,11 @@ class ModelButtonClick(DeactivableMixin, ModelSQL, ModelView): def reset(cls, model, names, records): assert all(r.__class__.__name__ == model for r in records) - clicks = [] - for records in grouped_slice(records): - clicks.extend(cls.search([ - ('button.model.name', '=', model), - ('button.name', 'in', names), - ('record_id', 'in', [r.id for r in records]), - ])) + clicks = cls.search([ + ('button.model.name', '=', model), + ('button.name', 'in', names), + ('record_id', 'in', records), + ]) cls.write(clicks, { 'active': False, }) @@ -1360,33 +1358,32 @@ class ModelData( def can_modify(cls, records, values): for Model, records in groupby( records, key=lambda r: r.__class__): - for sub_records in grouped_slice(records): - id2record = {r.id: r for r in sub_records} - domain = [ - ('model', '=', Model.__name__), - ('db_id', 'in', list(id2record.keys())), - ('noupdate', '=', False), - ] - if values is not None: - domain.append(('field_names', '!=', None)) - data = cls.search(domain, order=[]) - for data in data: - record = id2record[data.db_id] - if values is None: - raise AccessError( - gettext( - 'ir.msg_delete_xml_record', - **Model.__names__(record=record)), - gettext('ir.msg_base_config_record')) - else: - for field in values: - if field in data.field_names: - raise AccessError( - gettext( - 'ir.msg_write_xml_record', - **cls.__names__( - field=field, record=record)), - gettext('ir.msg_base_config_record')) + id2record = {r.id: r for r in records} + domain = [ + ('model', '=', Model.__name__), + ('db_id', 'in', list(id2record.keys())), + ('noupdate', '=', False), + ] + if values is not None: + domain.append(('field_names', '!=', None)) + data = cls.search(domain, order=[]) + for data in data: + record = id2record[data.db_id] + if values is None: + raise AccessError( + gettext( + 'ir.msg_delete_xml_record', + **Model.__names__(record=record)), + gettext('ir.msg_base_config_record')) + else: + for field in values: + if field in data.field_names: + raise AccessError( + gettext( + 'ir.msg_write_xml_record', + **cls.__names__( + field=field, record=record)), + gettext('ir.msg_base_config_record')) @classmethod @without_check_access @@ -1394,13 +1391,12 @@ class ModelData( data = [] for name, records in groupby( records, key=lambda r: r.__class__.__name__): - for sub_records in grouped_slice(records): - ids = [r.id for r in sub_records] - data += cls.search([ - ('model', '=', name), - ('db_id', 'in', ids), - ('noupdate', '=', True), - ], order=[]) + data += cls.search([ + ('model', '=', name), + ('db_id', 'in', records), + ('noupdate', '=', True), + ], + order=[]) cls.write(data, {'db_id': None}) @classmethod diff --git a/trytond/trytond/ir/module.py b/trytond/trytond/ir/module.py index fcf0558e86..741d2cc80b 100644 --- a/trytond/trytond/ir/module.py +++ b/trytond/trytond/ir/module.py @@ -13,7 +13,6 @@ from trytond.modules import get_module_info, get_modules from trytond.pool import Pool from trytond.pyson import Eval from trytond.rpc import RPC -from trytond.tools import grouped_slice from trytond.transaction import Transaction from trytond.wizard import ( Button, StateAction, StateTransition, StateView, Wizard) @@ -249,11 +248,10 @@ class Module(ModelSQL, ModelView): pool = Pool() Dependency = pool.get('ir.module.dependency') module_names = get_modules(with_test=Pool.test) - for sub_module_names in grouped_slice(module_names): - cls.delete(cls.search([ - ('state', '!=', 'activated'), - ('name', 'not in', list(sub_module_names)), - ])) + cls.delete(cls.search([ + ('state', '!=', 'activated'), + ('name', 'not in', module_names), + ])) modules = cls.search([]) name2module = {m.name: m for m in modules} @@ -303,12 +301,10 @@ class ModuleDependency(ModelSQL, ModelView): def get_state(cls, dependencies, name): pool = Pool() Module = pool.get('ir.module') - modules = [] names = [d.name for d in dependencies] - for sub_names in grouped_slice(names): - modules.extend(Module.search([ - ('name', 'in', list(sub_names)), - ])) + modules = Module.search([ + ('name', 'in', names), + ]) name2state = {m.name: m.state for m in modules} return {d.id: name2state.get(d.name, 'unknown') for d in dependencies} diff --git a/trytond/trytond/ir/note.py b/trytond/trytond/ir/note.py index 197d6429a0..171d5e6bfd 100644 --- a/trytond/trytond/ir/note.py +++ b/trytond/trytond/ir/note.py @@ -1,5 +1,6 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. +from collections import defaultdict from textwrap import TextWrapper from sql import Null @@ -9,7 +10,6 @@ from trytond.i18n import lazy_gettext from trytond.model import ModelSQL, ModelView, fields from trytond.pool import Pool from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction from .resource import ResourceMixin, resource_copy @@ -42,7 +42,7 @@ class Note(ResourceMixin, ModelSQL, ModelView): return '\n'.join(map(wrapper.fill, message.splitlines())) @classmethod - def get_unread(cls, ids, name): + def get_unread(cls, notes, name): pool = Pool() Read = pool.get('ir.note.read') cursor = Transaction().connection.cursor() @@ -50,17 +50,16 @@ class Note(ResourceMixin, ModelSQL, ModelView): table = cls.__table__() read = Read.__table__() - unread = {} - for sub_ids in grouped_slice(ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - query = table.join(read, 'LEFT', - condition=(table.id == read.note) - & (read.user == user_id) - ).select(table.id, - Case((read.user != Null, False), else_=True), - where=where) - cursor.execute(*query) - unread.update(cursor) + unread = defaultdict(bool) + query = table.join(read, 'LEFT', + condition=(table.id == read.note) + & (read.user == user_id) + ).select(table.id, + Case((read.user != Null, False), else_=True), + where=fields.SQL_OPERATORS['in']( + table.id, map(int, notes))) + cursor.execute(*query) + unread.update(cursor) return unread @classmethod @@ -90,12 +89,10 @@ class Note(ResourceMixin, ModelSQL, ModelView): if not value: Read.create([{'note': n.id, 'user': user_id} for n in notes]) else: - reads = [] - for sub_notes in grouped_slice(notes): - reads += Read.search([ - ('note', 'in', [n.id for n in sub_notes]), - ('user', '=', user_id), - ]) + reads = Read.search([ + ('note', 'in', notes), + ('user', '=', user_id), + ]) Read.delete(reads) @classmethod diff --git a/trytond/trytond/ir/queue_.py b/trytond/trytond/ir/queue_.py index 90738c6d6c..2b9a52227c 100644 --- a/trytond/trytond/ir/queue_.py +++ b/trytond/trytond/ir/queue_.py @@ -158,12 +158,9 @@ class Queue(ModelSQL): else: instances = None else: - ids = set() with inactive_records(): - for sub_ids in grouped_slice(instances): - records = Model.search([('id', 'in', list(sub_ids))]) - ids.update(map(int, records)) - if ids: + records = Model.search([('id', 'in', instances)]) + if ids := set(map(int, records)): instances = Model.browse( [i for i in instances if i in ids]) else: diff --git a/trytond/trytond/ir/resource.py b/trytond/trytond/ir/resource.py index b096251d8a..31efac75fa 100644 --- a/trytond/trytond/ir/resource.py +++ b/trytond/trytond/ir/resource.py @@ -10,7 +10,6 @@ from trytond.model import ( Index, Model, ModelSQL, ModelStorage, ModelView, fields) from trytond.pool import Pool from trytond.pyson import Eval -from trytond.tools import grouped_slice from trytond.transaction import Transaction, without_check_access __all__ = ['ResourceAccessMixin', 'ResourceMixin', 'resource_copy'] @@ -90,10 +89,9 @@ class ResourceAccessMixin(ModelStorage): record.resource.id) for RModel, ids in resources.items(): - for sub_ids in grouped_slice(ids): - allowed.update(RModel.search([ - ('id', 'in', list(sub_ids)), - ])) + allowed.update(RModel.search([ + ('id', 'in', ids), + ])) records = [ r for r in records diff --git a/trytond/trytond/ir/rule.py b/trytond/trytond/ir/rule.py index c28945c58c..b8e0e749d1 100644 --- a/trytond/trytond/ir/rule.py +++ b/trytond/trytond/ir/rule.py @@ -10,7 +10,6 @@ from trytond.model import Check, Index, ModelSQL, ModelView, fields from trytond.model.exceptions import AccessError, ValidationError from trytond.pool import Pool from trytond.pyson import Eval, If, PYSONDecoder -from trytond.tools import grouped_slice from trytond.transaction import Transaction, inactive_records @@ -346,18 +345,13 @@ class Rule(ModelSQL, ModelView): transaction = Transaction() def test_domain(ids, domain): - wrong_ids = [] # Use root to prevent infinite recursion with transaction.set_user(0, set_context=True), inactive_records(): - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - records = Model.search([ - ('id', 'in', sub_ids), - domain, - ], order=[]) - wrong_ids.extend( - set(sub_ids).difference(map(int, records))) - return wrong_ids + records = Model.search([ + ('id', 'in', ids), + domain, + ], order=[]) + return list(set(ids).difference(map(int, records))) domain = cls.domain_get(model_name, mode=mode) forbidden = test_domain(ids, domain) diff --git a/trytond/trytond/ir/translation.py b/trytond/trytond/ir/translation.py index 2f7d11bae9..5e53dab742 100644 --- a/trytond/trytond/ir/translation.py +++ b/trytond/trytond/ir/translation.py @@ -23,7 +23,7 @@ from trytond.i18n import gettext from trytond.model import Index, ModelSQL, ModelView, fields from trytond.pool import Pool from trytond.pyson import Eval, PYSONEncoder -from trytond.tools import cursor_dict, file_open, grouped_slice +from trytond.tools import cursor_dict, file_open from trytond.tools.string_ import LazyString, StringPartitioned from trytond.transaction import ( Transaction, inactive_records, without_check_access) @@ -493,16 +493,15 @@ class Translation( fuzzy_clause = [] else: fuzzy_clause = [('fuzzy', '=', False)] - for sub_to_fetch in grouped_slice(to_fetch): - for translation in cls.search([ - ('lang', '=', lang), - ('type', '=', ttype), - ('name', '=', name), - ('value', '!=', ''), - ('value', '!=', None), - ('res_id', 'in', list(sub_to_fetch)), - ] + fuzzy_clause): - translations[translation.res_id] = translation.value + for translation in cls.search([ + ('lang', '=', lang), + ('type', '=', ttype), + ('name', '=', name), + ('value', '!=', ''), + ('value', '!=', None), + ('res_id', 'in', to_fetch), + ] + fuzzy_clause): + translations[translation.res_id] = translation.value # Don't store fuzzy translation in cache if not fuzzy_translation: for res_id in to_fetch: @@ -519,15 +518,6 @@ class Translation( ModelFields = pool.get('ir.model.field') Model = pool.get('ir.model') Config = pool.get('ir.configuration') - transaction = Transaction() - in_max = transaction.database.IN_MAX - - if len(ids) > in_max: - for i in range(0, len(ids), in_max): - sub_ids = ids[i:i + in_max] - sub_values = values[i:i + in_max] - cls.set_ids(name, ttype, lang, sub_ids, sub_values) - return model_name, field_name = name.split(',') if model_name in ('ir.model.field', 'ir.model'): @@ -649,13 +639,11 @@ class Translation( @without_check_access def delete_ids(cls, model, ttype, ids): "Delete translation for each id" - translations = [] - for sub_ids in grouped_slice(ids): - translations += cls.search([ - ('type', '=', ttype), - ('name', 'like', model + ',%'), - ('res_id', 'in', list(sub_ids)), - ]) + translations = cls.search([ + ('type', '=', ttype), + ('name', 'like', model + ',%'), + ('res_id', 'in', ids), + ]) cls.delete(translations) @classmethod @@ -676,12 +664,6 @@ class Translation( parent_args = [] parent_langs = [] clauses = [] - transaction = Transaction() - if len(args) > transaction.database.IN_MAX: - for sub_args in grouped_slice(args): - res.update(cls.get_sources(list(sub_args))) - return res - to_cache = [] for name, ttype, lang, source in args: name = str(name) @@ -720,9 +702,10 @@ class Translation( res[(name, ttype, lang, source)] = parent_src[ (name, ttype, parent_lang, source)] - in_max = transaction.database.IN_MAX // 7 - for sub_clause in grouped_slice(clauses, in_max): - for translation in cls.search(['OR'] + list(sub_clause)): + while clauses: + clauses_slice = clauses[:100] + clauses = clauses[100:] + for translation in cls.search(['OR'] + clauses_slice): key = (translation.name, translation.type, translation.lang, translation.src) if key not in args: @@ -1524,11 +1507,10 @@ class TranslationClean(Wizard): with inactive_records(): for model_name, translations in records.items(): Model = pool.get(model_name) - for sub_ids in grouped_slice(translations.keys()): - sub_ids = list(sub_ids) - records = Model.search([('id', 'in', sub_ids)]) - for record_id in set(sub_ids) - set(map(int, records)): - to_delete.extend(translations[record_id]) + ids = translations.keys() + records = Model.search([('id', 'in', ids)]) + for record_id in set(ids) - set(map(int, records)): + to_delete.extend(translations[record_id]) # skip translation handled in ir.model.data models_data = ModelData.search([ diff --git a/trytond/trytond/ir/trigger.py b/trytond/trytond/ir/trigger.py index 4fb661339f..597f5d1cef 100644 --- a/trytond/trytond/ir/trigger.py +++ b/trytond/trytond/ir/trigger.py @@ -15,7 +15,6 @@ from trytond.model import ( from trytond.model.exceptions import ValidationError from trytond.pool import Pool from trytond.pyson import Eval, If, PYSONDecoder, TimeDelta -from trytond.tools import grouped_slice from trytond.transaction import Transaction @@ -204,20 +203,18 @@ class Trigger(DeactivableMixin, ModelSQL, ModelView): # Filter on limit_number if self.limit_number: new_ids = [] - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - where = fields.SQL_OPERATORS['in'](trigger_log.record_id, sub_ids) - cursor.execute(*trigger_log.select( - trigger_log.record_id, Count(), - where=where & (trigger_log.trigger == self.id), - group_by=trigger_log.record_id)) - number = dict(cursor) - for record_id in sub_ids: - if record_id not in number: - new_ids.append(record_id) - continue - if number[record_id] < self.limit_number: - new_ids.append(record_id) + where = fields.SQL_OPERATORS['in'](trigger_log.record_id, ids) + cursor.execute(*trigger_log.select( + trigger_log.record_id, Count(), + where=where & (trigger_log.trigger == self.id), + group_by=trigger_log.record_id)) + number = dict(cursor) + for record_id in ids: + if record_id not in number: + new_ids.append(record_id) + continue + if number[record_id] < self.limit_number: + new_ids.append(record_id) ids = new_ids # Filter on minimum_time_delay @@ -229,25 +226,23 @@ class Trigger(DeactivableMixin, ModelSQL, ModelView): now, = cursor.fetchone() if isinstance(now, str): now = datetime.datetime.fromisoformat(now) - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - where = fields.SQL_OPERATORS['in']( - trigger_log.record_id, sub_ids) - cursor.execute(*trigger_log.select( - trigger_log.record_id, Max(trigger_log.create_date), - where=where & (trigger_log.trigger == self.id), - group_by=trigger_log.record_id)) - delay = dict(cursor) - for record_id in sub_ids: - if record_id not in delay: - new_ids.append(record_id) - continue - # SQLite return string for MAX - if isinstance(delay[record_id], str): - delay[record_id] = datetime.datetime.fromisoformat( - delay[record_id]) - if now - delay[record_id] >= self.minimum_time_delay: - new_ids.append(record_id) + where = fields.SQL_OPERATORS['in']( + trigger_log.record_id, ids) + cursor.execute(*trigger_log.select( + trigger_log.record_id, Max(trigger_log.create_date), + where=where & (trigger_log.trigger == self.id), + group_by=trigger_log.record_id)) + delay = dict(cursor) + for record_id in ids: + if record_id not in delay: + new_ids.append(record_id) + continue + # SQLite return string for MAX + if isinstance(delay[record_id], str): + delay[record_id] = datetime.datetime.fromisoformat( + delay[record_id]) + if now - delay[record_id] >= self.minimum_time_delay: + new_ids.append(record_id) ids = new_ids records = Model.browse(ids) diff --git a/trytond/trytond/ir/ui/menu.py b/trytond/trytond/ir/ui/menu.py index 77eec57df6..a56324832a 100644 --- a/trytond/trytond/ir/ui/menu.py +++ b/trytond/trytond/ir/ui/menu.py @@ -7,7 +7,6 @@ from trytond.model import ( DeactivableMixin, ModelSQL, ModelView, fields, sequence_ordered, tree) from trytond.pool import Pool from trytond.rpc import RPC -from trytond.tools import grouped_slice from trytond.transaction import Transaction, inactive_records @@ -176,12 +175,10 @@ class UIMenu( menus = list(filter(has_action_access, menus)) - parent_ids = {x.parent.id for x in menus if x.parent} - parents = set() - for sub_parent_ids in grouped_slice(parent_ids): - parents.update(cls.search([ - ('id', 'in', list(sub_parent_ids)), - ])) + parent_ids = list({x.parent.id for x in menus if x.parent}) + parents = cls.search([ + ('id', 'in', parent_ids), + ]) # Re-browse to avoid side-cache access menus = cls.browse([x.id for x in menus if (x.parent and x.parent in parents) or not x.parent]) @@ -223,14 +220,10 @@ class UIMenu( def set_action(cls, menus, name, value): pool = Pool() ActionKeyword = pool.get('ir.action.keyword') - action_keywords = [] - transaction = Transaction() - for i in range(0, len(menus), transaction.database.IN_MAX): - sub_menus = menus[i:i + transaction.database.IN_MAX] - action_keywords += ActionKeyword.search([ - ('keyword', '=', 'tree_open'), - ('model', 'in', [str(menu) for menu in sub_menus]), - ]) + action_keywords = ActionKeyword.search([ + ('keyword', '=', 'tree_open'), + ('model', 'in', [str(menu) for menu in menus]), + ]) if action_keywords: with Transaction().set_context(_timestamp=False): ActionKeyword.delete(action_keywords) diff --git a/trytond/trytond/model/fields/binary.py b/trytond/trytond/model/fields/binary.py index d2529a775a..5bd5fbefd0 100644 --- a/trytond/trytond/model/fields/binary.py +++ b/trytond/trytond/model/fields/binary.py @@ -6,7 +6,7 @@ import logging from sql import Column, Null from trytond.filestore import filestore -from trytond.tools import cached_property, grouped_slice +from trytond.tools import cached_property from trytond.transaction import Transaction from .field import SQL_OPERATORS, Field @@ -83,19 +83,18 @@ class Binary(Field): def store_func(id, prefix): return self.cast(filestore.get(id, prefix=prefix)) - for sub_ids in grouped_slice(ids): - cursor.execute(*table.select( - table.id, Column(table, self.file_id), - where=SQL_OPERATORS['in'](table.id, sub_ids) - & (Column(table, self.file_id) != Null) - & (Column(table, self.file_id) != ''))) - for record_id, file_id in cursor: - try: - res[record_id] = store_func(file_id, prefix) - except (IOError, OSError): - logger.exception( - "failed to retrieve %r from filestore at %r", - file_id, prefix) + cursor.execute(*table.select( + table.id, Column(table, self.file_id), + where=SQL_OPERATORS['in'](table.id, ids) + & (Column(table, self.file_id) != Null) + & (Column(table, self.file_id) != ''))) + for record_id, file_id in cursor: + try: + res[record_id] = store_func(file_id, prefix) + except (IOError, OSError): + logger.exception( + "failed to retrieve %r from filestore at %r", + file_id, prefix) for i in values: if i['id'] in res: diff --git a/trytond/trytond/model/fields/dict.py b/trytond/trytond/model/fields/dict.py index 99fcc05bc9..34358081c2 100644 --- a/trytond/trytond/model/fields/dict.py +++ b/trytond/trytond/model/fields/dict.py @@ -8,7 +8,6 @@ from sql import Cast, CombiningQuery, Literal, Null, Select, operators from trytond import backend from trytond.pool import Pool from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder -from trytond.tools import grouped_slice from trytond.tools.immutabledict import ImmutableDict from trytond.transaction import Transaction @@ -202,11 +201,9 @@ class TranslatedDict(object): if self.type_ == 'values': domain = [('type_', '=', 'selection')] - records = [] - for key_names in grouped_slice(value.keys()): - records += SchemaModel.search([ - ('name', 'in', key_names), - ] + domain) + records = SchemaModel.search([ + ('name', 'in', value.keys()), + ] + domain) keys = SchemaModel.get_keys(records) if self.type_ == 'keys': diff --git a/trytond/trytond/model/fields/field.py b/trytond/trytond/model/fields/field.py index 4bb5393f2a..eb7d3944ca 100644 --- a/trytond/trytond/model/fields/field.py +++ b/trytond/trytond/model/fields/field.py @@ -806,8 +806,7 @@ class Translated: language = Transaction().language read_size = max(1, min( inst._cache.size_limit, - inst._local_cache.size_limit, - inst._transaction.database.IN_MAX)) + inst._local_cache.size_limit)) index = inst._ids.index(inst.id) ids = inst._ids[index:index + read_size] translations = Translation.get_ids( diff --git a/trytond/trytond/model/fields/fmany2one.py b/trytond/trytond/model/fields/fmany2one.py index de3f8fd3d3..c69da08e2c 100644 --- a/trytond/trytond/model/fields/fmany2one.py +++ b/trytond/trytond/model/fields/fmany2one.py @@ -3,6 +3,7 @@ from sql import Column, Literal +from trytond import backend from trytond.pool import Pool from trytond.tools import grouped_slice @@ -62,7 +63,8 @@ def fmany2one( values = set(filter(all, (tuple(getattr(r, s) for s in sources) for r in records))) values2id = {} - for sub_values in grouped_slice(values): + for sub_values in grouped_slice( + values, backend.MAX_QUERY_PARAMS // len(target_fields)): domain = ['OR'] for values in sub_values: domain.append( diff --git a/trytond/trytond/model/fields/many2many.py b/trytond/trytond/model/fields/many2many.py index b61be24210..fc88a781bc 100644 --- a/trytond/trytond/model/fields/many2many.py +++ b/trytond/trytond/model/fields/many2many.py @@ -2,14 +2,13 @@ # this repository contains the full copyright notices and license terms. import warnings from collections import defaultdict -from itertools import chain from sql import Literal, Null from sql.conditionals import Coalesce from trytond.pool import Pool from trytond.pyson import PYSONEncoder -from trytond.tools import cached_property, grouped_slice +from trytond.tools import cached_property from trytond.transaction import Transaction from .field import ( @@ -153,19 +152,15 @@ class Many2Many(Field): else: order += self.order - relations = [] - for sub_ids in grouped_slice(ids): - if reference_key: - references = ['%s,%s' % (model.__name__, x) for x in sub_ids] - clause = [(self.origin, 'in', references)] - else: - clause = [(self.origin, 'in', list(sub_ids))] - clause += [(self.target, '!=', None)] - if self.filter: - clause.append((self.target, 'where', self.filter)) - relations.append( - [r.id for r in Relation.search(clause, order=order)]) - to_read = list(chain(*relations)) + if reference_key: + references = ['%s,%s' % (model.__name__, x) for x in ids] + clause = [(self.origin, 'in', references)] + else: + clause = [(self.origin, 'in', ids)] + clause += [(self.target, '!=', None)] + if self.filter: + clause.append((self.target, 'where', self.filter)) + to_read = [r.id for r in Relation.search(clause, order=order)] relations = {t['id']: t for t in Relation.read(to_read, [self.origin, self.target])} @@ -229,19 +224,17 @@ class Many2Many(Field): target_to_delete.extend(Target.browse(target_ids)) def add(ids, target_ids): - target_ids = list(map(int, target_ids)) if not target_ids: return existing_ids = set() - for sub_ids in grouped_slice(target_ids): - relations = Relation.search([ - search_clause(ids), - (self.target, 'in', list(sub_ids)), - ]) - for relation in relations: - existing_ids.add(( - getattr(relation, self.origin).id, - getattr(relation, self.target).id)) + relations = Relation.search([ + search_clause(ids), + (self.target, 'in', target_ids), + ]) + for relation in relations: + existing_ids.add(( + getattr(relation, self.origin).id, + getattr(relation, self.target).id)) for new_id in target_ids: for record_id in ids: if (record_id, new_id) in existing_ids: @@ -252,14 +245,12 @@ class Many2Many(Field): }) def remove(ids, target_ids): - target_ids = list(map(int, target_ids)) if not target_ids: return - for sub_ids in grouped_slice(target_ids): - relation_to_delete.extend(Relation.search([ - search_clause(ids), - (self.target, 'in', list(sub_ids)), - ])) + relation_to_delete.extend(Relation.search([ + search_clause(ids), + (self.target, 'in', target_ids), + ])) def copy(ids, copy_ids, default=None): copy_ids = list(map(int, copy_ids)) diff --git a/trytond/trytond/model/fields/one2many.py b/trytond/trytond/model/fields/one2many.py index 0b16c18a9b..4559095fa1 100644 --- a/trytond/trytond/model/fields/one2many.py +++ b/trytond/trytond/model/fields/one2many.py @@ -2,7 +2,6 @@ # this repository contains the full copyright notices and license terms. import warnings from collections import defaultdict -from itertools import chain from sql import Literal from sql.conditionals import Coalesce @@ -11,7 +10,7 @@ from sql.operators import Exists import trytond.config as config from trytond.pool import Pool from trytond.pyson import PYSONEncoder -from trytond.tools import cached_property, grouped_slice +from trytond.tools import cached_property from trytond.transaction import Transaction from .field import ( @@ -156,17 +155,14 @@ class One2Many(Field): order += self.order elif Target._order: order += Target._order - targets = [] - for sub_ids in grouped_slice(ids): - if reference_key: - references = ['%s,%s' % (model.__name__, x) for x in sub_ids] - clause = [(self.field, 'in', references)] - else: - clause = [(self.field, 'in', list(sub_ids))] - if self.filter: - clause.append(self.filter) - targets.append([r.id for r in Target.search(clause, order=order)]) - to_read = list(chain(*targets)) + if reference_key: + references = ['%s,%s' % (model.__name__, x) for x in ids] + clause = [(self.field, 'in', references)] + else: + clause = [(self.field, 'in', ids)] + if self.filter: + clause.append(self.filter) + to_read = [r.id for r in Target.search(clause, order=order)] targets = {t['id']: t for t in Target.read(to_read, ['id', self.field])} @@ -235,7 +231,6 @@ class One2Many(Field): to_delete.extend(Target.browse(target_ids)) def add(ids, target_ids): - target_ids = list(map(int, target_ids)) if not target_ids: return targets = Target.browse(target_ids) @@ -249,17 +244,15 @@ class One2Many(Field): })) def remove(ids, target_ids): - target_ids = list(map(int, target_ids)) if not target_ids: return - for sub_ids in grouped_slice(target_ids): - targets = Target.search([ - search_clause(ids), - ('id', 'in', list(sub_ids)), - ]) - to_write.extend((targets, { - self.field: None, - })) + targets = Target.search([ + search_clause(ids), + ('id', 'in', target_ids), + ]) + to_write.extend((targets, { + self.field: None, + })) def copy(ids, copy_ids, default=None): copy_ids = list(map(int, copy_ids)) diff --git a/trytond/trytond/model/modelsql.py b/trytond/trytond/model/modelsql.py index ad488f3728..20eb9dd967 100644 --- a/trytond/trytond/model/modelsql.py +++ b/trytond/trytond/model/modelsql.py @@ -3,7 +3,7 @@ import datetime from collections import defaultdict from functools import wraps -from itertools import chain, groupby, islice, product, repeat +from itertools import groupby, islice, product, repeat from sql import ( Asc, Column, Desc, Expression, Literal, Null, NullsFirst, NullsLast, Table, @@ -753,16 +753,13 @@ class ModelSQL(ModelStorage): table = cls.__table_history__() user = User.__table__() - revisions = [] columns = [ Coalesce(table.write_date, table.create_date), table.id, user.name] - for sub_ids in grouped_slice(ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*table.join(user, 'LEFT', - Coalesce(table.write_uid, table.create_uid) == user.id) - .select(*columns, where=where, group_by=columns)) - revisions.append(cursor.fetchall()) - revisions = list(chain(*revisions)) + where = fields.SQL_OPERATORS['in'](table.id, ids) + cursor.execute(*table.join(user, 'LEFT', + Coalesce(table.write_uid, table.create_uid) == user.id) + .select(*columns, where=where, group_by=columns)) + revisions = list(cursor) revisions.sort(reverse=True) # SQLite uses char for COALESCE if revisions and isinstance(revisions[0][0], str): @@ -796,20 +793,20 @@ class ModelSQL(ModelStorage): continue columns.append(Column(table, fname)) hcolumns.append(Column(history, fname)) - for sub_ids in grouped_slice(ids): - if not deleted: - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*history.insert(hcolumns, - table.select(*columns, where=where))) - else: - if transaction.database.has_multirow_insert(): + if not deleted: + where = fields.SQL_OPERATORS['in'](table.id, ids) + cursor.execute(*history.insert(hcolumns, + table.select(*columns, where=where))) + else: + if transaction.database.has_multirow_insert(): + for sub_ids in grouped_slice(ids, backend.MAX_QUERY_PARAMS): cursor.execute(*history.insert(hcolumns, [[id_, CurrentTimestamp(), user] for id_ in sub_ids])) - else: - for id_ in sub_ids: - cursor.execute(*history.insert(hcolumns, - [[id_, CurrentTimestamp(), user]])) + else: + for id_ in ids: + cursor.execute(*history.insert(hcolumns, + [[id_, CurrentTimestamp(), user]])) @classmethod def _restore_history(cls, ids, datetime, _before=False): @@ -863,61 +860,58 @@ class ModelSQL(ModelStorage): window=Window([history.id], order_by=h_order) ).as_('__h_rank')) - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - if not _before: - hwhere = (column_datetime <= datetime) + if not _before: + hwhere = (column_datetime <= datetime) + else: + hwhere = (column_datetime < datetime) + + hwhere &= fields.SQL_OPERATORS['in'](history.id, ids) + history_select.query.where = hwhere + + cursor.execute(*history_values) + for values in cursor.fetchall(): + if is_deleted(values): + to_delete.add(values[id_idx]) else: - hwhere = (column_datetime < datetime) + to_update.add(values[id_idx]) + to_delete |= (deleted_sub_ids := set(ids) - to_update) - hwhere &= fields.SQL_OPERATORS['in'](history.id, sub_ids) - history_select.query.where = hwhere + # we need to skip the deleted IDs that are all None history records + # because they could fail the UPDATE + if to_delete: + history_select.query.where &= ( + fields.SQL_OPERATORS['not in']( + history.id, list(deleted_sub_ids))) - cursor.execute(*history_values) - for values in cursor.fetchall(): - if is_deleted(values): - to_delete.add(values[id_idx]) - else: - to_update.add(values[id_idx]) - to_delete |= (deleted_sub_ids := set(sub_ids) - to_update) - - # we need to skip the deleted IDs that are all None history records - # because they could fail the UPDATE - if to_delete: - history_select.query.where &= ( - fields.SQL_OPERATORS['not in']( - history.id, list(deleted_sub_ids))) - - # Some of the sub_ids are not updated because they are not in the - # table anymore, they should be undeleted from the value in the - # history table - to_undelete = set() - update_query = table.update( - columns, history_columns, - from_=[history_values], - where=history_values.id == table.id) - if transaction.database.has_returning(): - update_query.returning = [table.id] - cursor.execute(*update_query) - to_undelete.update(to_update - set(r[0] for r in cursor)) - else: - cursor.execute(*table - .right_join(history_values, history_values.id == table.id) - .select(history_values.id, where=table.id == Null)) - to_undelete.update(r[0] for r in cursor) - cursor.execute(*update_query) - if to_undelete: - history_select.query.where = (hwhere - & fields.SQL_OPERATORS['in']( - history.id, list(to_undelete))) - cursor.execute(*table.insert( - columns, - history_values.select(*history_columns))) + # Some of the ids are not updated because they are not in the + # table anymore, they should be undeleted from the value in the + # history table + to_undelete = set() + update_query = table.update( + columns, history_columns, + from_=[history_values], + where=history_values.id == table.id) + if transaction.database.has_returning(): + update_query.returning = [table.id] + cursor.execute(*update_query) + to_undelete.update(to_update - set(r[0] for r in cursor)) + else: + cursor.execute(*table + .right_join(history_values, history_values.id == table.id) + .select(history_values.id, where=table.id == Null)) + to_undelete.update(r[0] for r in cursor) + cursor.execute(*update_query) + if to_undelete: + history_select.query.where = (hwhere + & fields.SQL_OPERATORS['in']( + history.id, list(to_undelete))) + cursor.execute(*table.insert( + columns, + history_values.select(*history_columns))) if to_delete: - for sub_ids in grouped_slice(to_delete): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*table.delete(where=where)) + cursor.execute(*table.delete( + where=fields.SQL_OPERATORS['in'](table.id, ids))) cls._insert_history(list(to_delete), True) if to_update: cls._insert_history(list(to_update)) @@ -939,7 +933,7 @@ class ModelSQL(ModelStorage): table = cls.__table__() if not transaction.timestamp: return - for sub_ids in grouped_slice(ids): + for sub_ids in grouped_slice(ids, backend.MAX_QUERY_PARAMS // 3): where = Or() for id_ in sub_ids: try: @@ -970,7 +964,6 @@ class ModelSQL(ModelStorage): def create(cls, vlist): transaction = Transaction() cursor = transaction.connection.cursor() - in_max = transaction.database.IN_MAX pool = Pool() Translation = pool.get('ir.translation') @@ -987,7 +980,8 @@ class ModelSQL(ModelStorage): if transaction.database.has_multirow_insert(): vlist = ( s for s in grouped_slice( - vlist, in_max // (len(column_names) or 1))) + vlist, + backend.MAX_QUERY_PARAMS // (len(column_names) + 1))) else: vlist = ([v] for v in vlist) @@ -1188,14 +1182,12 @@ class ModelSQL(ModelStorage): result = [] table = cls.__table__() - in_max = transaction.database.IN_MAX history_order = None history_clause = None history_limit = None if (cls._history and transaction.context.get('_datetime') and not cls._is_table_query()): - in_max = 1 table = cls.__table_history__() column = Coalesce(table.write_date, table.create_date) history_clause = (column <= Transaction().context['_datetime']) @@ -1255,8 +1247,7 @@ class ModelSQL(ModelStorage): tables, dom_exp = cls.search_domain( domain, active_test=False, tables=tables) from_ = convert_from(None, tables) - for sub_ids in grouped_slice(ids, in_max): - sub_ids = list(sub_ids) + for sub_ids in ([i] for i in ids) if history_clause else [ids]: red_sql = fields.SQL_OPERATORS['in'](table.id, sub_ids) where = red_sql if history_clause: @@ -1517,26 +1508,25 @@ class ModelSQL(ModelStorage): columns.append(Column(table, fname)) update_values.append(field.sql_format(value)) - for sub_ids in grouped_slice(ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - try: - cursor.execute(*table.update( - columns, update_values, where=where)) - except ( - backend.DatabaseIntegrityError, - backend.DatabaseDataError) as exception: - transaction = Transaction() - with Transaction().new_transaction(): - if isinstance( - exception, backend.DatabaseIntegrityError): - cls.__raise_integrity_error( - exception, values, list(values.keys()), - transaction=transaction) - elif isinstance(exception, backend.DatabaseDataError): - cls.__raise_data_error( - exception, values, list(values.keys()), - transaction=transaction) - raise + try: + cursor.execute(*table.update( + columns, update_values, + where=fields.SQL_OPERATORS['in'](table.id, ids))) + except ( + backend.DatabaseIntegrityError, + backend.DatabaseDataError) as exception: + transaction = Transaction() + with Transaction().new_transaction(): + if isinstance( + exception, backend.DatabaseIntegrityError): + cls.__raise_integrity_error( + exception, values, list(values.keys()), + transaction=transaction) + elif isinstance(exception, backend.DatabaseDataError): + cls.__raise_data_error( + exception, values, list(values.keys()), + transaction=transaction) + raise for fname, value in values.items(): field = cls._fields[fname] @@ -1575,7 +1565,6 @@ class ModelSQL(ModelStorage): @no_table_query def delete(cls, records): transaction = Transaction() - in_max = transaction.database.IN_MAX cursor = transaction.connection.cursor() pool = Pool() @@ -1590,12 +1579,11 @@ class ModelSQL(ModelStorage): tree_ids = {} for fname in cls._mptt_fields: - tree_ids[fname] = [] - for sub_ids in grouped_slice(ids): - where = fields.SQL_OPERATORS['in']( - Column(table, fname), sub_ids) - cursor.execute(*table.select(table.id, where=where)) - tree_ids[fname] += [x[0] for x in cursor] + cursor.execute(*table.select( + table.id, + where=fields.SQL_OPERATORS['in']( + Column(table, fname), ids))) + tree_ids[fname] = [x[0] for x in cursor] foreign_keys_tocheck = [] foreign_keys_toupdate = [] @@ -1617,73 +1605,65 @@ class ModelSQL(ModelStorage): else: foreign_keys_tocheck.append((model, field_name)) - if len(records) > in_max: - # Clean self referencing foreign keys - # before deleting them by small groups - # Use the record id as value instead of NULL - # in case the field is required - foreign_fields_to_clean = [ - fn for m, fn in foreign_keys_tocheck if m == cls] - if foreign_fields_to_clean: - for sub_ids in grouped_slice(ids): - columns = [ - Column(table, n) for n in foreign_fields_to_clean] - cursor.execute(*table.update( - columns, [table.id] * len(foreign_fields_to_clean), - where=fields.SQL_OPERATORS['in'](table.id, sub_ids) - )) + # Clean self referencing foreign keys + # before deleting them by small groups + # Use the record id as value instead of NULL + # in case the field is required + foreign_fields_to_clean = [ + fn for m, fn in foreign_keys_tocheck if m == cls] + if foreign_fields_to_clean: + columns = [ + Column(table, n) for n in foreign_fields_to_clean] + cursor.execute(*table.update( + columns, [table.id] * len(foreign_fields_to_clean), + where=fields.SQL_OPERATORS['in'](table.id, ids) + )) - def get_related_records(Model, field_name, sub_ids): + def get_related_records(Model, field_name, ids): if issubclass(Model, ModelSQL): foreign_table = Model.__table__() foreign_red_sql = fields.SQL_OPERATORS['in']( - Column(foreign_table, field_name), sub_ids) + Column(foreign_table, field_name), ids) cursor.execute(*foreign_table.select(foreign_table.id, where=foreign_red_sql)) related_records = Model.browse([x[0] for x in cursor]) else: with without_check_access(), inactive_records(): related_records = Model.search( - [(field_name, 'in', sub_ids)], + [(field_name, 'in', ids)], order=[]) if Model == cls: related_records = list(set(related_records) - set(records)) return related_records - for sub_ids, sub_records in zip( - grouped_slice(ids), grouped_slice(records)): - sub_ids = list(sub_ids) - red_sql = fields.SQL_OPERATORS['in'](table.id, sub_ids) + for Model, field_name in foreign_keys_toupdate: + related_records = get_related_records(Model, field_name, ids) + if related_records: + Model.write(related_records, { + field_name: None, + }) - for Model, field_name in foreign_keys_toupdate: - related_records = get_related_records( - Model, field_name, sub_ids) - if related_records: - Model.write(related_records, { - field_name: None, - }) + for Model, field_name in foreign_keys_todelete: + related_records = get_related_records(Model, field_name, ids) + if related_records: + Model.delete(related_records) - for Model, field_name in foreign_keys_todelete: - related_records = get_related_records( - Model, field_name, sub_ids) - if related_records: - Model.delete(related_records) + for Model, field_name in foreign_keys_tocheck: + if get_related_records(Model, field_name, ids): + error_args = Model.__names__(field_name) + raise ForeignKeyError( + gettext('ir.msg_foreign_model_exist', + **error_args)) - for Model, field_name in foreign_keys_tocheck: - if get_related_records(Model, field_name, sub_ids): - error_args = Model.__names__(field_name) - raise ForeignKeyError( - gettext('ir.msg_foreign_model_exist', - **error_args)) - - try: - cursor.execute(*table.delete(where=red_sql)) - except backend.DatabaseIntegrityError as exception: - transaction = Transaction() - with Transaction().new_transaction(): - cls.__raise_integrity_error( - exception, {}, transaction=transaction) - raise + try: + cursor.execute(*table.delete( + where=fields.SQL_OPERATORS['in'](table.id, ids))) + except backend.DatabaseIntegrityError as exception: + transaction = Transaction() + with Transaction().new_transaction(): + cls.__raise_integrity_error( + exception, {}, transaction=transaction) + raise cls._insert_history(ids, deleted=True) @@ -1704,14 +1684,12 @@ class ModelSQL(ModelStorage): User = Group = None table = cls.__table__() transaction = Transaction() - in_max = transaction.database.IN_MAX history_clause = None limit = None if (mode == 'read' and cls._history and transaction.context.get('_datetime') and not cls._is_table_query()): - in_max = 1 table = cls.__table_history__() column = Coalesce(table.write_date, table.create_date) history_clause = (column <= Transaction().context['_datetime']) @@ -1726,7 +1704,7 @@ class ModelSQL(ModelStorage): tables, dom_exp = cls.search_domain( domain, active_test=False, tables=tables) from_ = convert_from(None, tables) - for sub_ids in grouped_slice(ids, in_max): + for sub_ids in ([i] for i in ids) if history_clause else [ids]: sub_ids = set(sub_ids) where = fields.SQL_OPERATORS['in'](table.id, sub_ids) if history_clause: @@ -1988,30 +1966,26 @@ class ModelSQL(ModelStorage): return select cursor.execute(*select) - rows = list(cursor_dict(cursor, transaction.database.IN_MAX)) + rows = list(cursor_dict(cursor)) cache = transaction.get_cache() delete_records = transaction.delete_records[cls.__name__] - # Can not cache the history value if we are not sure to have fetch all - # the rows for each records - if (not (cls._history and transaction.context.get('_datetime')) - or len(rows) < transaction.database.IN_MAX): - keys = None - for data in islice(rows, 0, cache.size_limit): - if data['id'] in delete_records: - continue - if keys is None: - keys = list(data.keys()) - for k in keys[:]: - if k in ('_timestamp', '_datetime', '__id'): - continue - field = cls._fields[k] - if not getattr(field, 'datetime_field', None): - keys.remove(k) - continue - for k in keys: - del data[k] - cache[cls.__name__][data['id']]._update(data) + keys = None + for data in islice(rows, 0, cache.size_limit): + if data['id'] in delete_records: + continue + if keys is None: + keys = list(data.keys()) + for k in keys[:]: + if k in ('_timestamp', '_datetime', '__id'): + continue + field = cls._fields[k] + if not getattr(field, 'datetime_field', None): + keys.remove(k) + continue + for k in keys: + del data[k] + cache[cls.__name__][data['id']]._update(data) return cls.browse([x['id'] for x in rows]) @@ -2115,10 +2089,9 @@ class ModelSQL(ModelStorage): [Concat(Concat(Coalesce( parent.select(parent.path, where=parent.id == parent_column), - ''), table.id), '/')]) - for sub_ids in grouped_slice(ids): - query.where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*query) + ''), table.id), '/')], + where=fields.SQL_OPERATORS['in'](table.id, ids)) + cursor.execute(*query) @classmethod def _update_path(cls, field_names, list_ids): @@ -2128,9 +2101,9 @@ class ModelSQL(ModelStorage): table = cls.__table__() parent = cls.__table__() - def update_path(query, column, sub_ids): + def update_path(query, column, ids): updated = set() - query.where = fields.SQL_OPERATORS['in'](table.id, sub_ids) + query.where = fields.SQL_OPERATORS['in'](table.id, ids) cursor.execute(*query) for old_path, new_path in cursor: if old_path == new_path: @@ -2156,10 +2129,8 @@ class ModelSQL(ModelStorage): .select(path_column, Concat(Concat( Coalesce(parent_path_column, ''), table.id), '/'))) - for sub_ids in grouped_slice(ids): - sub_ids = list(sub_ids) - while not update_path(query, path_column, sub_ids): - pass + while not update_path(query, path_column, ids): + pass @classmethod def _update_mptt(cls, field_names, list_ids, values=None): @@ -2294,50 +2265,47 @@ class ModelSQL(ModelStorage): else: columns = list(sql.columns) columns.insert(0, table.id) - in_max = transaction.database.IN_MAX // (len(columns) + 1) - for sub_ids in grouped_slice(ids, in_max): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - if isinstance(sql, Exclude) and sql.where: - where &= sql.where + where = fields.SQL_OPERATORS['in'](table.id, ids) + if isinstance(sql, Exclude) and sql.where: + where &= sql.where - cursor.execute(*table.select(*columns, where=where)) + cursor.execute(*table.select(*columns, where=where)) - where = Literal(False) - for row in cursor: - row = list(row) - clause = table.id != row.pop(0) - if not database.has_range(): - values = [] - for col in sql.columns: - if isinstance(col, Range): - range_ = col.__class__( - row.pop(0), row.pop(0), row.pop(0)) - values.append(range_) - else: - values.append(row.pop(0)) - else: - values = row - for column, operator, value in zip( - sql.columns, sql.operators, values): - if value is None: - # NULL is always unique - clause &= Literal(False) - clause &= operator(column, value) - where |= clause - if isinstance(sql, Exclude) and sql.where: - where &= sql.where - cursor.execute( - *table.select(table.id, where=where, limit=1)) - if cursor.fetchone(): - raise SQLConstraintError(gettext(error)) + where = Literal(False) + for row in cursor: + row = list(row) + clause = table.id != row.pop(0) + if not database.has_range(): + values = [] + for col in sql.columns: + if isinstance(col, Range): + range_ = col.__class__( + row.pop(0), row.pop(0), row.pop(0)) + values.append(range_) + else: + values.append(row.pop(0)) + else: + values = row + for column, operator, value in zip( + sql.columns, sql.operators, values): + if value is None: + # NULL is always unique + clause &= Literal(False) + clause &= operator(column, value) + where |= clause + if isinstance(sql, Exclude) and sql.where: + where &= sql.where + cursor.execute( + *table.select(table.id, where=where, limit=1)) + if cursor.fetchone(): + raise SQLConstraintError(gettext(error)) elif isinstance(sql, Check): - for sub_ids in grouped_slice(ids): - where = fields.SQL_OPERATORS['in'](table.id, sub_ids) - cursor.execute(*table.select(table.id, - where=~sql.expression & where, - limit=1)) - if cursor.fetchone(): - raise SQLConstraintError(gettext(error)) + cursor.execute(*table.select(table.id, + where=~sql.expression + & fields.SQL_OPERATORS['in'](table.id, ids), + limit=1)) + if cursor.fetchone(): + raise SQLConstraintError(gettext(error)) @dualmethod def lock(cls, records=None): diff --git a/trytond/trytond/model/modelstorage.py b/trytond/trytond/model/modelstorage.py index b8069a6001..f77f075bef 100644 --- a/trytond/trytond/model/modelstorage.py +++ b/trytond/trytond/model/modelstorage.py @@ -832,12 +832,11 @@ class ModelStorage(Model): ids = dict.fromkeys(map(int, records)) with without_check_access(): - for sub_records in grouped_slice(records): - for record in ModelData.search([ - ('model', '=', cls.__name__), - ('db_id', 'in', [r.id for r in sub_records]), - ]): - ids[record.db_id] = f'{record.module}.{record.fs_id}' + for record in ModelData.search([ + ('model', '=', cls.__name__), + ('db_id', 'in', records), + ]): + ids[record.db_id] = f'{record.module}.{record.fs_id}' return ids @classmethod @@ -1453,33 +1452,20 @@ class ModelStorage(Model): domains[context][domain].append(record) # Select strategy depending if it is closer to one domain per # record or one domain for all records - # Do not use IN_MAX to let spaces for the pyson domain - in_max = Transaction().database.IN_MAX - count = in_max // 10 for context, ctx_domains in domains.items(): if (not dict_domain and len(ctx_domains) > len(records) * 0.5): - new_domains = {} - for sub_domains in grouped_slice( - list(ctx_domains.keys()), count): - grouped_domain = ['OR'] - grouped_records = [] - for d in sub_domains: - sub_records = ctx_domains[d] - grouped_records.extend(sub_records) - relations = relation_domain(field, sub_records) - if len(relations) > in_max: - break - grouped_domain.append( - [('id', 'in', - [r.id for r in relations]), d]) - else: - new_domains[freeze(grouped_domain)] = \ - grouped_records - continue - break - else: - domains[context] = new_domains + grouped_domain = ['OR'] + grouped_records = [] + for d, sub_records in ctx_domains.items(): + grouped_records.extend(sub_records) + relations = relation_domain(field, sub_records) + grouped_domain.append( + [('id', 'in', + [r.id for r in relations]), d]) + domains[context] = { + freeze(grouped_domain): grouped_records, + } else: domains[freeze(field.context)][freeze(field.domain)].extend( records) @@ -1517,68 +1503,67 @@ class ModelStorage(Model): def validate_relation_domain(field, records, Relation, domain): relations = relation_domain(field, records) - if relations: - for sub_relations in grouped_slice(relations): - sub_relations = set(sub_relations) - finds = Relation.search(['AND', - [('id', 'in', [r.id for r in sub_relations])], - domain, - ]) - invalid_relations = sub_relations - set(finds) - if Relation == cls and field._type not in { - 'many2one', 'one2many', 'many2many', 'one2one', - 'reference'}: - invalid_records = invalid_relations - elif field._type.endswith('2many'): - invalid_records = [ - r for r in records - if invalid_relations.intersection( - getattr(r, field.name))] - else: - invalid_records = [ - r for r in records - if getattr(r, field.name) in invalid_relations] - if invalid_records: - invalid_record = invalid_records.pop() - if invalid_relations == invalid_records: - invalid_relation = invalid_record - else: - invalid_relation = getattr( - invalid_record, field.name) - domain = field.domain - if is_pyson(domain): - domain = _record_eval_pyson(records[0], domain) - if isinstance(domain, dict): - domain = domain.get(Relation.__name__, []) - msg = gettext( - 'ir.msg_domain_validation_record', - **cls.__names__(field.name, invalid_record)) - fields = set() - level = 0 - if field not in Relation._fields.values(): - expression = domain_parse(domain) - for variable in expression.variables: - parts = variable.split('.') - fields.add(parts[0]) - level = max(level, len(parts)) - else: - fields.add(field.name) - for field_name in sorted(fields): - env = EvalEnvironment(invalid_relation, Relation) - invalid_domain = domain_inversion( - domain, field_name, env) - if isinstance(invalid_domain, bool): - continue - if (len(fields) > 1 # no need to evaluate - and eval_domain(invalid_domain, env)): - continue - field_def = Relation.fields_get( - [field_name], level=level) - raise DomainValidationError( - msg, domain=(invalid_domain, field_def)) - field_def = Relation.fields_get(fields, level=level) - raise DomainValidationError( - msg, domain=(domain, field_def)) + if not relations: + return + finds = Relation.search(['AND', + [('id', 'in', relations)], + domain, + ]) + invalid_relations = set(relations) - set(finds) + if Relation == cls and field._type not in { + 'many2one', 'one2many', 'many2many', 'one2one', + 'reference'}: + invalid_records = invalid_relations + elif field._type.endswith('2many'): + invalid_records = [ + r for r in records + if invalid_relations.intersection( + getattr(r, field.name))] + else: + invalid_records = [ + r for r in records + if getattr(r, field.name) in invalid_relations] + if invalid_records: + invalid_record = invalid_records.pop() + if invalid_relations == invalid_records: + invalid_relation = invalid_record + else: + invalid_relation = getattr( + invalid_record, field.name) + domain = field.domain + if is_pyson(domain): + domain = _record_eval_pyson(records[0], domain) + if isinstance(domain, dict): + domain = domain.get(Relation.__name__, []) + msg = gettext( + 'ir.msg_domain_validation_record', + **cls.__names__(field.name, invalid_record)) + fields = set() + level = 0 + if field not in Relation._fields.values(): + expression = domain_parse(domain) + for variable in expression.variables: + parts = variable.split('.') + fields.add(parts[0]) + level = max(level, len(parts)) + else: + fields.add(field.name) + for field_name in sorted(fields): + env = EvalEnvironment(invalid_relation, Relation) + invalid_domain = domain_inversion( + domain, field_name, env) + if isinstance(invalid_domain, bool): + continue + if (len(fields) > 1 # no need to evaluate + and eval_domain(invalid_domain, env)): + continue + field_def = Relation.fields_get( + [field_name], level=level) + raise DomainValidationError( + msg, domain=(invalid_domain, field_def)) + field_def = Relation.fields_get(fields, level=level) + raise DomainValidationError( + msg, domain=(domain, field_def)) if field_names is None: field_names = cls._fields.keys() @@ -1960,8 +1945,7 @@ class ModelStorage(Model): s.add(id_) yield id_ read_size = max(1, min( - self._cache.size_limit, self._local_cache.size_limit, - self._transaction.database.IN_MAX)) + self._cache.size_limit, self._local_cache.size_limit)) index = self._ids.index(self.id) ids = islice(self._ids, index, index + read_size) ids = unique(filter(filter_, ids)) diff --git a/trytond/trytond/modules/__init__.py b/trytond/trytond/modules/__init__.py index 88aaba48b3..95f667c832 100644 --- a/trytond/trytond/modules/__init__.py +++ b/trytond/trytond/modules/__init__.py @@ -198,10 +198,10 @@ def load_module_graph(graph, pool, update=None, lang=None, indexes=None): with transaction.connection.cursor() as cursor: modules = [x.name for x in graph] module2state = defaultdict(lambda: 'not activated') - for sub_modules in tools.grouped_slice(modules): - cursor.execute(*ir_module.select(ir_module.name, ir_module.state, - where=ir_module.name.in_(list(sub_modules)))) - module2state.update(cursor) + cursor.execute(*ir_module.select( + ir_module.name, ir_module.state, + where=ir_module.name.in_(modules))) + module2state.update(cursor) modules = set(modules) new_modules = modules - module2state.keys() diff --git a/trytond/trytond/res/group.py b/trytond/trytond/res/group.py index f577b47981..f129e952bf 100644 --- a/trytond/trytond/res/group.py +++ b/trytond/trytond/res/group.py @@ -7,7 +7,6 @@ from sql import With from trytond.model import ( DeactivableMixin, ModelSQL, ModelView, Unique, fields, tree) from trytond.pool import Pool -from trytond.tools import grouped_slice class MenuMany2Many(fields.Many2Many): @@ -17,12 +16,9 @@ class MenuMany2Many(fields.Many2Many): res = super().get(ids, model, name, values=values) menu_ids = list(set(chain(*res.values()))) - test_ids = [] - for sub_ids in grouped_slice(menu_ids): - test_ids.append(list(map(int, Menu.search([ - ('id', 'in', sub_ids), - ])))) - menu_ids = set(chain(*test_ids)) + menu_ids = list(map(int, Menu.search([ + ('id', 'in', menu_ids), + ]))) for group_id, ids in res.items(): res[group_id] = tuple(id_ for id_ in ids if id_ in menu_ids) return res diff --git a/trytond/trytond/res/user.py b/trytond/trytond/res/user.py index 9d1418cfa6..4b86222146 100644 --- a/trytond/trytond/res/user.py +++ b/trytond/trytond/res/user.py @@ -42,7 +42,6 @@ from trytond.pyson import Bool, Eval, PYSONEncoder from trytond.report import Report, get_email from trytond.rpc import RPC from trytond.sendmail import send_message_transactional -from trytond.tools import grouped_slice from trytond.tools.email_ import ( EmailNotValidError, normalize_email, set_from_header, validate_email) from trytond.transaction import Transaction, without_check_access @@ -319,16 +318,16 @@ class User(avatar_mixin(100, 'login'), DeactivableMixin, ModelSQL, ModelView): seconds=config.getint('session', 'max_age')) result = dict((u.id, 0) for u in users) with without_check_access(): - for sub_ids in grouped_slice(users): - sessions = Session.search([ - ('create_uid', 'in', sub_ids), - ], order=[('create_uid', 'ASC')]) + sessions = Session.search([ + ('create_uid', 'in', users), + ], + order=[('create_uid', 'ASC')]) - def filter_(session): - return abs(session.last_modified_at - now) < timeout - result.update(dict((i, len(list(g))) - for i, g in groupby(filter(filter_, sessions), - attrgetter('create_uid.id')))) + def filter_(session): + return abs(session.last_modified_at - now) < timeout + result.update(dict((i, len(list(g))) + for i, g in groupby(filter(filter_, sessions), + attrgetter('create_uid.id')))) return result @classmethod @@ -882,10 +881,9 @@ class UserDevice(ModelSQL): @classmethod def clear(cls, logins): - for sub_logins in grouped_slice(logins): - cls.delete(cls.search([ - ('login', 'in', list(sub_logins)), - ])) + cls.delete(cls.search([ + ('login', 'in', logins), + ])) class UserAction(ModelSQL): diff --git a/trytond/trytond/tests/test_history.py b/trytond/trytond/tests/test_history.py index ce71167c50..98a44e56a8 100644 --- a/trytond/trytond/tests/test_history.py +++ b/trytond/trytond/tests/test_history.py @@ -541,54 +541,51 @@ class HistoryTestCase(TestCase): [l.name for l in history.lines_at_stamp], ['a', 'b']) @with_transaction() - def test_search_cursor_max(self): - 'Test search with number of history entries at database.IN_MAX' + def test_search_large_number(self): + 'Test search with large number of history entries' pool = Pool() History = pool.get('test.history') transaction = Transaction() - database = transaction.database history = History(value=-1) history.save() - for history.value in range(database.IN_MAX + 1): + for history.value in range(1_000 + 1): history.save() with transaction.set_context(_datetime=datetime.datetime.max): record, = History.search([]) - self.assertEqual(record.value, database.IN_MAX) + self.assertEqual(record.value, 1_000) @with_transaction() - def test_search_cursor_max_entries(self): - 'Test search for skipping first history entries at database.IN_MAX' + def test_search_large_number_entries(self): + 'Test search for skipping first history entries of large number' pool = Pool() History = pool.get('test.history') transaction = Transaction() - database = transaction.database for i in range(0, 2): history = History(value=-1) history.save() - for history.value in range(database.IN_MAX + 1): + for history.value in range(1_000 + 1): history.save() with transaction.set_context(_datetime=datetime.datetime.max): records = History.search([]) - self.assertEqual({r.value for r in records}, {database.IN_MAX}) + self.assertEqual({r.value for r in records}, {1_000}) self.assertEqual(len(records), 2) @with_transaction() - def test_search_cursor_max_histories(self): - 'Test search with number of histories at database.IN_MAX' + def test_search_large_number_histories(self): + 'Test search with large number of histories' pool = Pool() History = pool.get('test.history') transaction = Transaction() - database = transaction.database - n = database.IN_MAX + 1 + n = 1_000 History.create([{'value': 1}] * n) with transaction.set_context(_datetime=datetime.datetime.max): diff --git a/trytond/trytond/tests/test_modelsql.py b/trytond/trytond/tests/test_modelsql.py index 3b9712a2fd..cbc48288c9 100644 --- a/trytond/trytond/tests/test_modelsql.py +++ b/trytond/trytond/tests/test_modelsql.py @@ -742,11 +742,6 @@ class ModelSQLTestCase(TestCase): pool = Pool() Model = pool.get('test.modelsql.fk.tree') - # Delete all records at a time - in_max = Transaction().database.IN_MAX - self.addCleanup(setattr, Transaction().database, 'IN_MAX', in_max) - Transaction().database.IN_MAX = 2 - parent = Model() parent.save() child = Model(parent_restrict=parent) @@ -760,11 +755,6 @@ class ModelSQLTestCase(TestCase): pool = Pool() Model = pool.get('test.modelsql.fk.tree') - # Delete one record at a time - in_max = Transaction().database.IN_MAX - self.addCleanup(setattr, Transaction().database, 'IN_MAX', in_max) - Transaction().database.IN_MAX = 1 - parent = Model() parent.save() child = Model(parent_restrict=parent) diff --git a/trytond/trytond/tests/test_modelstorage.py b/trytond/trytond/tests/test_modelstorage.py index bbef489f3f..b025c13b95 100644 --- a/trytond/trytond/tests/test_modelstorage.py +++ b/trytond/trytond/tests/test_modelstorage.py @@ -400,29 +400,6 @@ class ModelStorageTestCase(TestCase): self.assertTrue(cm.exception.domain[0]) self.assertTrue(cm.exception.domain[1]['value']) - @with_transaction() - def test_pyson_domain_unique_in_max(self): - "Test unique pyson domain validation with greater IN_MAX" - pool = Pool() - Model = pool.get('test.modelstorage.pyson_domain') - - in_max = Transaction().database.IN_MAX - self.addCleanup(setattr, Transaction().database, 'IN_MAX', in_max) - Transaction().database.IN_MAX = 1 - - # Use modulo 6 so len(domains) is greater then len(records) * 0.5 - # and more than 1 (IN_MAX) have the same domain - Model.create( - [{'constraint': str(i % 6), 'value': str(i % 6)} - for i in range(10)]) - - with self.assertRaises(DomainValidationError) as cm: - Model.create( - [{'constraint': str(i % 6), 'value': str(i)} - for i in range(10)]) - self.assertTrue(cm.exception.domain[0]) - self.assertTrue(cm.exception.domain[1]['value']) - @with_transaction() def test_pyson_domain_single(self): "Test pyson domain validation for 1 record" diff --git a/trytond/trytond/tests/test_tools.py b/trytond/trytond/tests/test_tools.py index a429659933..2e4bceede5 100644 --- a/trytond/trytond/tests/test_tools.py +++ b/trytond/trytond/tests/test_tools.py @@ -27,10 +27,10 @@ from trytond.pool import Pool from trytond.tests.test_tryton import TestCase from trytond.tools import ( cached_property, decimal_, email_, escape_wildcard, file_open, firstline, - grouped_slice, is_full_text, is_instance_method, likify, lstrip_wildcard, - pair, pairwise_longest, reduce_domain, remove_forbidden_chars, - rstrip_wildcard, slugify, sortable_values, sqlite_apply_types, - strip_wildcard, timezone, unescape_wildcard, unpair) + is_full_text, is_instance_method, likify, lstrip_wildcard, pair, + pairwise_longest, reduce_domain, remove_forbidden_chars, rstrip_wildcard, + slugify, sortable_values, sqlite_apply_types, strip_wildcard, timezone, + unescape_wildcard, unpair) from trytond.tools.chart import sparkline from trytond.tools.domain_inversion import ( canonicalize, concat, domain_inversion, eval_domain, @@ -83,23 +83,6 @@ class ToolsTestCase(TestCase): self.assertEqual(reduce_domain(i), j, '%s -> %s != %s' % (i, reduce_domain(i), j)) - def test_grouped_slice(self): - "Test grouped slice" - for (values, count, result) in [ - (list(range(10)), 5, [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]), - (list(range(5)), 5, [[0, 1, 2, 3, 4]]), - (list(range(5)), 2, [[0, 1], [2, 3], [4]]), - ]: - with self.subTest(values=values, count=count): - self.assertEqual( - list(map(list, grouped_slice(values, count))), result) - - def test_grouped_slice_generator(self): - "Test grouped slice" - self.assertEqual( - list(map(list, grouped_slice((x for x in range(10)), 5))), - [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]) - def test_pairwise_longest(self): "Test pairwise_longest" self.assertEqual( diff --git a/trytond/trytond/tools/misc.py b/trytond/trytond/tools/misc.py index 24d95afe49..f9c6fda536 100644 --- a/trytond/trytond/tools/misc.py +++ b/trytond/trytond/tools/misc.py @@ -149,9 +149,11 @@ def reduce_domain(domain): def grouped_slice(records, count=None): 'Grouped slice' - from trytond.transaction import Transaction if count is None: - count = Transaction().database.IN_MAX + warnings.warn( + "grouped_slice without size is deprecated", + DeprecationWarning) + count = 1 count = max(1, count) if not isinstance(records, Sized): records = list(records)