diff --git a/modules/account/account.py b/modules/account/account.py
index 6f17f816ac..3257baf87a 100644
--- a/modules/account/account.py
+++ b/modules/account/account.py
@@ -22,8 +22,7 @@ from trytond.pool import Pool
from trytond.pyson import Bool, Eval, Id, If, PYSONEncoder
from trytond.report import Report
from trytond.tools import (
- grouped_slice, is_full_text, lstrip_wildcard, pair, sql_pairing,
- sqlite_apply_types)
+ is_full_text, lstrip_wildcard, pair, sql_pairing, sqlite_apply_types)
from trytond.transaction import Transaction, check_access, inactive_records
from trytond.wizard import (
Button, StateAction, StateTransition, StateView, Wizard)
@@ -1018,27 +1017,26 @@ class Account(
ids = [a.id for a in c_accounts]
with transaction.set_context(company=company.id):
line_query, fiscalyear_ids = MoveLine.query_get(line)
- for sub_ids in grouped_slice(ids):
- red_sql = fields.SQL_OPERATORS['in'](table_a.id, sub_ids)
- if context.get('flat_balance'):
- query = (table_a
- .join(line, condition=line.account == table_a.id))
- else:
- query = (table_a
- .join(table_c,
- condition=(table_c.left >= table_a.left)
- & (table_c.right <= table_a.right))
- .join(line, condition=line.account == table_c.id))
- query = query.select(
- table_a.id,
- Sum(Coalesce(line.debit, 0) - Coalesce(line.credit, 0)
- ).as_('balance'),
- where=red_sql & line_query,
- group_by=table_a.id)
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- balances.update(dict(cursor))
+ red_sql = fields.SQL_OPERATORS['in'](table_a.id, ids)
+ if context.get('flat_balance'):
+ query = (table_a
+ .join(line, condition=line.account == table_a.id))
+ else:
+ query = (table_a
+ .join(table_c,
+ condition=(table_c.left >= table_a.left)
+ & (table_c.right <= table_a.right))
+ .join(line, condition=line.account == table_c.id))
+ query = query.select(
+ table_a.id,
+ Sum(Coalesce(line.debit, 0) - Coalesce(line.credit, 0)
+ ).as_('balance'),
+ where=red_sql & line_query,
+ group_by=table_a.id)
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ balances.update(dict(cursor))
for account in c_accounts:
balances[account.id] = account.currency.round(
@@ -1091,20 +1089,19 @@ class Account(
columns.append(
Sum(Coalesce(Column(line, name), 0)).as_(name))
types.append('NUMERIC')
- for sub_ids in grouped_slice(ids):
- red_sql = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- query = (table.join(line, 'LEFT',
- condition=line.account == table.id
- ).select(*columns,
- where=red_sql & line_query,
- group_by=table.id))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, types)
- cursor.execute(*query)
- for row in cursor:
- account_id = row[0]
- for i, name in enumerate(names, 1):
- result[name][account_id] = row[i]
+ where = fields.SQL_OPERATORS['in'](table.id, ids)
+ query = (table.join(line, 'LEFT',
+ condition=line.account == table.id
+ ).select(*columns,
+ where=where & line_query,
+ group_by=table.id))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, types)
+ cursor.execute(*query)
+ for row in cursor:
+ account_id = row[0]
+ for i, name in enumerate(names, 1):
+ result[name][account_id] = row[i]
for account in c_accounts:
for name in names:
if name == 'line_count':
@@ -1161,14 +1158,12 @@ class Account(
if fiscalyear.state == 'closed' and deferral:
Deferral = pool.get(deferral)
id2deferral = {}
- ids = [a.id for a in accounts]
- for sub_ids in grouped_slice(ids):
- deferrals = Deferral.search([
- ('fiscalyear', '=', fiscalyear.id),
- ('account', 'in', list(sub_ids)),
- ])
- for deferral in deferrals:
- id2deferral[deferral.account.id] = deferral
+ deferrals = Deferral.search([
+ ('fiscalyear', '=', fiscalyear),
+ ('account', 'in', accounts),
+ ])
+ for deferral in deferrals:
+ id2deferral[deferral.account.id] = deferral
for account in accounts:
if account.id in id2deferral:
@@ -1246,26 +1241,24 @@ class Account(
if field_names and not (field_names & {'closed', 'type'}):
return
accounts = [a for a in accounts if a.closed or not a.type]
- for sub_accounts in grouped_slice(accounts):
- sub_accounts = list(sub_accounts)
- if Line.search([
- ('account', 'in', [a.id for a in sub_accounts]),
- ], order=[], limit=1):
- for account in sub_accounts:
- if not account.closed:
- continue
- lines = Line.search([
- ('account', '=', account.id),
- ], order=[], limit=1)
- if lines:
- if account.closed:
- raise AccountValidationError(gettext(
- 'account.msg_account_closed_lines',
- account=account.rec_name))
- elif account.type:
- raise AccountValidationError(gettext(
- 'account.msg_account_no_type_lines',
- account=account.rec_name))
+ if Line.search([
+ ('account', 'in', accounts),
+ ], order=[], limit=1):
+ for account in accounts:
+ if not account.closed:
+ continue
+ lines = Line.search([
+ ('account', '=', account.id),
+ ], order=[], limit=1)
+ if lines:
+ if account.closed:
+ raise AccountValidationError(gettext(
+ 'account.msg_account_closed_lines',
+ account=account.rec_name))
+ elif account.type:
+ raise AccountValidationError(gettext(
+ 'account.msg_account_no_type_lines',
+ account=account.rec_name))
@classmethod
def copy(cls, accounts, default=None):
@@ -1530,34 +1523,30 @@ class AccountParty(ActivePeriodMixin, ModelSQL):
party_ids = {a.party.id for a in c_records}
with transaction.set_context(company=company.id):
line_query, fiscalyear_ids = MoveLine.query_get(line)
- for sub_account_ids in grouped_slice(account_ids):
- account_sql = fields.SQL_OPERATORS['in'](
- table_a.id, sub_account_ids)
- for sub_party_ids in grouped_slice(party_ids):
- party_sql = fields.SQL_OPERATORS['in'](
- line.party, sub_party_ids)
- if context.get('flat_balance'):
- query = (table_a
- .join(line, condition=line.account == table_a.id))
- else:
- query = (table_a
- .join(table_c,
- condition=(table_c.left >= table_a.left)
- & (table_c.right <= table_a.right))
- .join(line, condition=line.account == table_c.id))
- query = query.select(
- table_a.id,
- line.party,
- Sum(
- Coalesce(line.debit, 0)
- - Coalesce(line.credit, 0)).as_('balance'),
- where=account_sql & party_sql & line_query,
- group_by=[table_a.id, line.party])
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, None, 'NUMERIC'])
- cursor.execute(*query)
- for account_id, party_id, balance in cursor:
- balances[pair(party_id, account_id)] = balance
+ account_sql = fields.SQL_OPERATORS['in'](table_a.id, account_ids)
+ party_sql = fields.SQL_OPERATORS['in'](line.party, party_ids)
+ if context.get('flat_balance'):
+ query = (table_a
+ .join(line, condition=line.account == table_a.id))
+ else:
+ query = (table_a
+ .join(table_c,
+ condition=(table_c.left >= table_a.left)
+ & (table_c.right <= table_a.right))
+ .join(line, condition=line.account == table_c.id))
+ query = query.select(
+ table_a.id,
+ line.party,
+ Sum(
+ Coalesce(line.debit, 0)
+ - Coalesce(line.credit, 0)).as_('balance'),
+ where=account_sql & party_sql & line_query,
+ group_by=[table_a.id, line.party])
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, None, 'NUMERIC'])
+ cursor.execute(*query)
+ for account_id, party_id, balance in cursor:
+ balances[pair(party_id, account_id)] = balance
for record in c_records:
balances[record.id] = record.currency.round(
@@ -1609,24 +1598,20 @@ class AccountParty(ActivePeriodMixin, ModelSQL):
with transaction.set_context(company=company.id):
line_query, fiscalyear_ids = MoveLine.query_get(line)
- for sub_account_ids in grouped_slice(account_ids):
- account_sql = fields.SQL_OPERATORS['in'](
- table.id, sub_account_ids)
- for sub_party_ids in grouped_slice(party_ids):
- party_sql = fields.SQL_OPERATORS['in'](
- line.party, sub_party_ids)
- query = (table.join(line, 'LEFT',
- condition=line.account == table.id
- ).select(*columns,
- where=account_sql & party_sql & line_query,
- group_by=[table.id, line.party]))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, types)
- cursor.execute(*query)
- for row in cursor:
- id_ = pair(*row[0:2])
- for i, name in enumerate(names, 2):
- result[name][id_] = row[i]
+ account_sql = fields.SQL_OPERATORS['in'](table.id, account_ids)
+ party_sql = fields.SQL_OPERATORS['in'](line.party, party_ids)
+ query = (table.join(line, 'LEFT',
+ condition=line.account == table.id
+ ).select(*columns,
+ where=account_sql & party_sql & line_query,
+ group_by=[table.id, line.party]))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, types)
+ cursor.execute(*query)
+ for row in cursor:
+ id_ = pair(*row[0:2])
+ for i, name in enumerate(names, 2):
+ result[name][id_] = row[i]
for record in c_records:
for name in names:
if name == 'line_count':
@@ -1753,26 +1738,25 @@ class AccountDeferral(ModelSQL, ModelView):
account_child = Account.__table__()
balances = defaultdict(Decimal)
- for sub_deferrals in grouped_slice(deferrals):
- red_sql = fields.SQL_OPERATORS['in'](
- table.id, [d.id for d in sub_deferrals])
- query = (table
- .join(account, condition=table.account == account.id)
- .join(account_child,
- condition=(account_child.left >= account.left)
- & (account_child.right <= account.right))
- .join(table_child,
- condition=(table_child.account == account_child.id)
- & (table_child.fiscalyear == table.fiscalyear))
- .select(
- table.id.as_('id'),
- Sum(table_child.debit - table_child.credit).as_('balance'),
- where=red_sql,
- group_by=table.id))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- balances.update(dict(cursor))
+ where = fields.SQL_OPERATORS['in'](
+ table.id, [d.id for d in deferrals])
+ query = (table
+ .join(account, condition=table.account == account.id)
+ .join(account_child,
+ condition=(account_child.left >= account.left)
+ & (account_child.right <= account.right))
+ .join(table_child,
+ condition=(table_child.account == account_child.id)
+ & (table_child.fiscalyear == table.fiscalyear))
+ .select(
+ table.id.as_('id'),
+ Sum(table_child.debit - table_child.credit).as_('balance'),
+ where=where,
+ group_by=table.id))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ balances.update(dict(cursor))
return balances
def get_currency(self, name):
@@ -2490,25 +2474,22 @@ class GeneralLedgerLine(DescriptionOriginMixin, ModelSQL, ModelView):
party_ids.add(r.party.id)
query = account_party.select(
- account_party.account, account_party.party, account_party.id)
- for sub_account_ids in grouped_slice(account_ids):
- account_where = fields.SQL_OPERATORS['in'](
- account_party.account, sub_account_ids)
- for sub_party_ids in grouped_slice(party_ids):
- query.where = (account_where
- & fields.SQL_OPERATORS['in'](
- account_party.party, sub_party_ids))
- cursor.execute(*query)
- for account, party, id_ in cursor:
- key = (account, party)
- try:
- account_party_ids = account_party2ids[key]
- except KeyError:
- # There can be more combinations of account-party in
- # the database than from records
- continue
- for record_id in account_party_ids:
- account_parties[record_id] = id_
+ account_party.account, account_party.party, account_party.id,
+ where=fields.SQL_OPERATORS['in'](
+ account_party.account, account_ids)
+ & fields.SQL_OPERATORS['in'](
+ account_party.party, party_ids))
+ cursor.execute(*query)
+ for account, party, id_ in cursor:
+ key = (account, party)
+ try:
+ account_party_ids = account_party2ids[key]
+ except KeyError:
+ # There can be more combinations of account-party in
+ # the database than from records
+ continue
+ for record_id in account_party_ids:
+ account_parties[record_id] = id_
return account_parties
@classmethod
diff --git a/modules/account/company.py b/modules/account/company.py
index 8eaad4a99e..e387db76c8 100644
--- a/modules/account/company.py
+++ b/modules/account/company.py
@@ -12,7 +12,7 @@ from trytond.model import fields
from trytond.model.exceptions import AccessError
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
@@ -65,26 +65,24 @@ class Company(metaclass=PoolMeta):
else:
code = name
today_where = Literal(True)
- for sub_companies in grouped_slice(companies):
- sub_ids = [p.id for p in sub_companies]
- company_where = fields.SQL_OPERATORS['in'](
- account.company, sub_ids)
- query = (line
- .join(account,
- condition=account.id == line.account)
- .join(account_type,
- condition=account.type == account_type.id)
- .select(account.company.as_('id'), amount.as_(name),
- where=(getattr(account_type, code)
- & (line.reconciliation == Null)
- & company_where
- & today_where),
- group_by=account.company))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- for company_id, value in cursor:
- amounts[name][company_id] = value.quantize(exp[company_id])
+ company_where = fields.SQL_OPERATORS['in'](
+ account.company, map(int, companies))
+ query = (line
+ .join(account,
+ condition=account.id == line.account)
+ .join(account_type,
+ condition=account.type == account_type.id)
+ .select(account.company.as_('id'), amount.as_(name),
+ where=(getattr(account_type, code)
+ & (line.reconciliation == Null)
+ & company_where
+ & today_where),
+ group_by=account.company))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ for company_id, value in cursor:
+ amounts[name][company_id] = value.quantize(exp[company_id])
return amounts
@classmethod
diff --git a/modules/account/journal.py b/modules/account/journal.py
index 29c388f71f..5aa3f03a8a 100644
--- a/modules/account/journal.py
+++ b/modules/account/journal.py
@@ -13,8 +13,7 @@ from trytond.model.exceptions import AccessError
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Eval
-from trytond.tools import (
- grouped_slice, is_full_text, lstrip_wildcard, sqlite_apply_types)
+from trytond.tools import is_full_text, lstrip_wildcard, sqlite_apply_types
from trytond.transaction import Transaction
STATES = {
@@ -111,29 +110,27 @@ class Journal(
& (move.date <= context.get('end_date'))
& ~Coalesce(account_type.receivable, False)
& ~Coalesce(account_type.payable, False)
- & (move.company == company.id))
- for sub_journals in grouped_slice(journals):
- sub_journals = list(sub_journals)
- red_sql = fields.SQL_OPERATORS['in'](
- move.journal, [j.id for j in sub_journals])
- query = line.join(move, 'LEFT', condition=line.move == move.id
- ).join(account, 'LEFT', condition=line.account == account.id
- ).join(account_type, 'LEFT',
- condition=account.type == account_type.id
- ).select(
- move.journal,
- Sum(line.debit).as_('debit'),
- Sum(line.credit).as_('credit'),
- where=where & red_sql,
- group_by=move.journal)
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC', 'NUMERIC'])
- cursor.execute(*query)
- for journal_id, debit, credit in cursor:
- result['debit'][journal_id] = company.currency.round(debit)
- result['credit'][journal_id] = company.currency.round(credit)
- result['balance'][journal_id] = company.currency.round(
- debit - credit)
+ & (move.company == company.id)
+ & fields.SQL_OPERATORS['in'](
+ move.journal, [j.id for j in journals]))
+ query = line.join(move, 'LEFT', condition=line.move == move.id
+ ).join(account, 'LEFT', condition=line.account == account.id
+ ).join(account_type, 'LEFT',
+ condition=account.type == account_type.id
+ ).select(
+ move.journal,
+ Sum(line.debit).as_('debit'),
+ Sum(line.credit).as_('credit'),
+ where=where,
+ group_by=move.journal)
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC', 'NUMERIC'])
+ cursor.execute(*query)
+ for journal_id, debit, credit in cursor:
+ result['debit'][journal_id] = company.currency.round(debit)
+ result['credit'][journal_id] = company.currency.round(credit)
+ result['balance'][journal_id] = company.currency.round(
+ debit - credit)
return result
@classmethod
@@ -154,16 +151,15 @@ class Journal(
super().check_modification(
mode, journals, values=values, external=external)
if mode == 'write' and 'type' in values:
- for sub_journals in grouped_slice(journals):
- moves = Move.search([
- ('journal', 'in', [j.id for j in sub_journals]),
- ('state', '=', 'posted')
- ], order=[], limit=1)
- if moves:
- move, = moves
- raise AccessError(gettext(
- 'account.msg_journal_account_moves',
- journal=move.journal.rec_name))
+ moves = Move.search([
+ ('journal', 'in', [j.id for j in journals]),
+ ('state', '=', 'posted')
+ ], order=[], limit=1)
+ if moves:
+ move, = moves
+ raise AccessError(gettext(
+ 'account.msg_journal_account_moves',
+ journal=move.journal.rec_name))
class JournalCashContext(ModelView):
diff --git a/modules/account/move.py b/modules/account/move.py
index c88dac1b31..d75ec99ee8 100644
--- a/modules/account/move.py
+++ b/modules/account/move.py
@@ -21,7 +21,7 @@ from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If, PYSONEncoder
from trytond.report import Report
from trytond.rpc import RPC
-from trytond.tools import firstline, grouped_slice, sqlite_apply_types
+from trytond.tools import firstline, sqlite_apply_types
from trytond.transaction import Transaction, check_access
from trytond.wizard import (
Button, StateAction, StateTransition, StateView, Wizard)
@@ -361,33 +361,32 @@ class Move(DescriptionOriginMixin, ModelSQL, ModelView):
for company, moves in groupby(moves, key=lambda m: m.company):
currency = company.currency
- for sub_moves in grouped_slice(list(moves)):
- red_sql = fields.SQL_OPERATORS['in'](
- line.move, [m.id for m in sub_moves])
+ where = fields.SQL_OPERATORS['in'](
+ line.move, [m.id for m in moves])
- valid_move_query = line.select(
- line.move,
- where=red_sql,
- group_by=line.move,
- having=Abs(Round(
- Sum(line.debit - line.credit),
- currency.digits)) < abs(currency.rounding))
- cursor.execute(*line.update(
- [line.state],
- ['valid'],
- where=line.move.in_(valid_move_query)))
+ valid_move_query = line.select(
+ line.move,
+ where=where,
+ group_by=line.move,
+ having=Abs(Round(
+ Sum(line.debit - line.credit),
+ currency.digits)) < abs(currency.rounding))
+ cursor.execute(*line.update(
+ [line.state],
+ ['valid'],
+ where=line.move.in_(valid_move_query)))
- draft_move_query = line.select(
- line.move,
- where=red_sql,
- group_by=line.move,
- having=Abs(Round(
- Sum(line.debit - line.credit),
- currency.digits)) >= abs(currency.rounding))
- cursor.execute(*line.update(
- [line.state],
- ['draft'],
- where=line.move.in_(draft_move_query)))
+ draft_move_query = line.select(
+ line.move,
+ where=where,
+ group_by=line.move,
+ having=Abs(Round(
+ Sum(line.debit - line.credit),
+ currency.digits)) >= abs(currency.rounding))
+ cursor.execute(*line.update(
+ [line.state],
+ ['draft'],
+ where=line.move.in_(draft_move_query)))
Transaction().counter += 1
for cache in Transaction().cache.values():
@@ -460,53 +459,49 @@ class Move(DescriptionOriginMixin, ModelSQL, ModelView):
today = Date.today()
currency = company.currency
c_moves = list(c_moves)
- for sub_moves in grouped_slice(c_moves):
- sub_moves_ids = [m.id for m in sub_moves]
+ moves_ids = [m.id for m in moves]
- cursor.execute(*move.select(
- move.id,
- where=fields.SQL_OPERATORS['in'](
- move.id, sub_moves_ids)
- & ~Exists(line.select(
- line.move,
- where=line.move == move.id))))
- try:
- move_id, = cursor.fetchone()
- except TypeError:
- pass
- else:
- raise PostError(
- gettext('account.msg_post_empty_move',
- move=cls(move_id).rec_name))
+ cursor.execute(*move.select(
+ move.id,
+ where=fields.SQL_OPERATORS['in'](move.id, moves_ids)
+ & ~Exists(line.select(
+ line.move,
+ where=line.move == move.id))))
+ try:
+ move_id, = cursor.fetchone()
+ except TypeError:
+ pass
+ else:
+ raise PostError(
+ gettext('account.msg_post_empty_move',
+ move=cls(move_id).rec_name))
- cursor.execute(*line.select(
- line.move,
- where=fields.SQL_OPERATORS['in'](
- line.move, sub_moves_ids),
- group_by=line.move,
- having=Abs(Round(
- Sum(line.debit - line.credit),
- currency.digits)) >= abs(currency.rounding)))
- try:
- move_id, = cursor.fetchone()
- except TypeError:
- pass
- else:
- raise PostError(
- gettext('account.msg_post_unbalanced_move',
- move=cls(move_id).rec_name))
+ cursor.execute(*line.select(
+ line.move,
+ where=fields.SQL_OPERATORS['in'](line.move, moves_ids),
+ group_by=line.move,
+ having=Abs(Round(
+ Sum(line.debit - line.credit),
+ currency.digits)) >= abs(currency.rounding)))
+ try:
+ move_id, = cursor.fetchone()
+ except TypeError:
+ pass
+ else:
+ raise PostError(
+ gettext('account.msg_post_unbalanced_move',
+ move=cls(move_id).rec_name))
- cursor.execute(*line.select(
- line.id,
- where=fields.SQL_OPERATORS['in'](
- line.move, sub_moves_ids)
- & (line.debit == Decimal(0))
- & (line.credit == Decimal(0))
- & (line.reconciliation == Null)
- & ((line.amount_second_currency == Null)
- | (line.amount_second_currency == Decimal(0)))
- ))
- to_reconcile.extend(l for l, in cursor)
+ cursor.execute(*line.select(
+ line.id,
+ where=fields.SQL_OPERATORS['in'](line.move, moves_ids)
+ & (line.debit == Decimal(0))
+ & (line.credit == Decimal(0))
+ & (line.reconciliation == Null)
+ & ((line.amount_second_currency == Null)
+ | (line.amount_second_currency == Decimal(0)))
+ ))
+ to_reconcile.extend(l for l, in cursor)
missing_number = defaultdict(list)
for move in c_moves:
@@ -832,63 +827,62 @@ class MoveLineMixin:
date = Coalesce(line.maturity_date, move.date)
for company, lines in groupby(lines, lambda l: l.company):
- for sub_lines in grouped_slice(lines):
- sub_lines = list(sub_lines)
- id2currency = {
- l.id: l.second_currency or company.currency
- for l in sub_lines}
- where = account.company == company.id
- where_type = Literal(False)
- if context.get('receivable', True):
- where_type |= account_type.receivable
- if context.get('payable', True):
- where_type |= account_type.payable
- where &= where_type
- if not context.get('reconciled'):
- if hasattr(cls, 'reconciliation'):
- where &= line.reconciliation == Null
- elif hasattr(cls, 'reconciled'):
- where &= ~line.reconciled
+ lines = list(lines)
+ id2currency = {
+ l.id: l.second_currency or company.currency
+ for l in lines}
+ where = account.company == company.id
+ where_type = Literal(False)
+ if context.get('receivable', True):
+ where_type |= account_type.receivable
+ if context.get('payable', True):
+ where_type |= account_type.payable
+ where &= where_type
+ if not context.get('reconciled'):
+ if hasattr(cls, 'reconciliation'):
+ where &= line.reconciliation == Null
+ elif hasattr(cls, 'reconciled'):
+ where &= ~line.reconciled
- currency = Coalesce(line.second_currency, company.currency.id)
- sign = Case(
- (account_type.statement == 'income', -1),
- else_=1)
- balance = sign * Case(
- (line.amount_second_currency != Null,
- line.amount_second_currency),
- else_=line.debit - line.credit)
- balance = Sum(balance,
- window=Window(
- [line.party, currency],
- order_by=[date.asc.nulls_first, line.id.desc]))
+ currency = Coalesce(line.second_currency, company.currency.id)
+ sign = Case(
+ (account_type.statement == 'income', -1),
+ else_=1)
+ balance = sign * Case(
+ (line.amount_second_currency != Null,
+ line.amount_second_currency),
+ else_=line.debit - line.credit)
+ balance = Sum(balance,
+ window=Window(
+ [line.party, currency],
+ order_by=[date.asc.nulls_first, line.id.desc]))
- party_where = Literal(False)
- parties = {l.party for l in sub_lines}
- if None in parties:
- party_where |= line.party == parties.discard(None)
- party_where |= fields.SQL_OPERATORS['in'](
- line.party, map(int, parties))
- where &= party_where
+ party_where = Literal(False)
+ parties = {l.party for l in lines}
+ if None in parties:
+ party_where |= line.party == parties.discard(None)
+ party_where |= fields.SQL_OPERATORS['in'](
+ line.party, map(int, parties))
+ where &= party_where
- query = (line
- .join(move, condition=line.move == move.id)
- .join(account, condition=line.account == account.id)
- .join(
- account_type,
- condition=account.type == account_type.id)
- .select(
- line.id.as_('id'), balance.as_('balance'),
- where=where))
- query = query.select(
- query.id, query.balance.as_('balance'),
- where=fields.SQL_OPERATORS['in'](
- query.id, [l.id for l in sub_lines]))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- balances.update(
- (i, id2currency[i].round(a)) for (i, a) in cursor)
+ query = (line
+ .join(move, condition=line.move == move.id)
+ .join(account, condition=line.account == account.id)
+ .join(
+ account_type,
+ condition=account.type == account_type.id)
+ .select(
+ line.id.as_('id'), balance.as_('balance'),
+ where=where))
+ query = query.select(
+ query.id, query.balance.as_('balance'),
+ where=fields.SQL_OPERATORS['in'](
+ query.id, [l.id for l in lines]))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ balances.update(
+ (i, id2currency[i].round(a)) for (i, a) in cursor)
return balances
def get_rec_name(self, name):
diff --git a/modules/account/party.py b/modules/account/party.py
index 0b082fa4d6..3e744eee56 100644
--- a/modules/account/party.py
+++ b/modules/account/party.py
@@ -16,7 +16,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.modules.party.exceptions import EraseError
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval, If
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.tools import timezone as tz
from trytond.transaction import Transaction
@@ -200,14 +200,13 @@ class Party(CompanyMultiValueMixin, metaclass=PoolMeta):
company = user.company
currency = company.currency
- for sub_parties in grouped_slice(parties):
- query = cls._receivable_payable_query(names, company, sub_parties)
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None] + ['NUMERIC'] * 4)
- cursor.execute(*query)
- for party_id, *values in cursor:
- for name, value in zip(names, values):
- amounts[name][party_id] = currency.round(value)
+ query = cls._receivable_payable_query(names, company, parties)
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None] + ['NUMERIC'] * 4)
+ cursor.execute(*query)
+ for party_id, *values in cursor:
+ for name, value in zip(names, values):
+ amounts[name][party_id] = currency.round(value)
return amounts
@classmethod
diff --git a/modules/account/period.py b/modules/account/period.py
index 9d327202f2..e607010bea 100644
--- a/modules/account/period.py
+++ b/modules/account/period.py
@@ -4,6 +4,7 @@
from sql import Null
from sql.operators import Equal, NotEqual
+from trytond import backend
from trytond.cache import Cache
from trytond.const import OPERATORS
from trytond.i18n import gettext
@@ -226,7 +227,8 @@ class Period(Workflow, ModelSQL, ModelView):
return
lang = Lang.get()
- for sub_periods in grouped_slice(periods):
+ for sub_periods in grouped_slice(
+ periods, backend.MAX_QUERY_PARAMS // 3):
domain = ['OR']
for period in sub_periods:
domain.append([
diff --git a/modules/account_asset/asset.py b/modules/account_asset/asset.py
index aa0e4b4c57..6ed0e52813 100644
--- a/modules/account_asset/asset.py
+++ b/modules/account_asset/asset.py
@@ -16,7 +16,7 @@ from trytond.modules.company import CompanyReport
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If
-from trytond.tools import cached_property, grouped_slice
+from trytond.tools import cached_property
from trytond.transaction import Transaction, check_access, without_check_access
from trytond.wizard import (
Button, StateReport, StateTransition, StateView, Wizard)
@@ -559,13 +559,11 @@ class Asset(Workflow, ModelSQL, ModelView):
cls.create_lines(assets)
moves = []
- lines = []
- for asset_ids in grouped_slice(assets):
- lines += Line.search([
- ('asset', 'in', list(asset_ids)),
- ('date', '<=', date),
- ('move', '=', None),
- ])
+ lines = Line.search([
+ ('asset', 'in', assets),
+ ('date', '<=', date),
+ ('move', '=', None),
+ ])
for line in lines:
moves.append(line.asset.get_move(line))
Move.save(moves)
diff --git a/modules/account_budget/account.py b/modules/account_budget/account.py
index 77858c0b0e..19b7936764 100644
--- a/modules/account_budget/account.py
+++ b/modules/account_budget/account.py
@@ -17,7 +17,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If
from trytond.rpc import RPC
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
from trytond.wizard import (
Button, StateAction, StateTransition, StateView, Wizard)
@@ -55,12 +55,11 @@ class AmountMixin:
balances = defaultdict(Decimal)
for keys, grouped_records in groupby(
records, key=cls._get_amount_group_key):
- for sub_records in grouped_slice(list(grouped_records)):
- query = cls._get_amount_query(sub_records, dict(keys))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- balances.update(cursor)
+ query = cls._get_amount_query(list(grouped_records), dict(keys))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ balances.update(cursor)
total = {n: {} for n in names}
for record in records:
@@ -212,10 +211,9 @@ class BudgetLineMixin(
if backend.name == 'sqlite':
sqlite_apply_types(query, [None, 'NUMERIC'])
- for sub_ids in grouped_slice(ids):
- query.where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*query)
- amounts.update(cursor)
+ query.where = fields.SQL_OPERATORS['in'](table.id, ids)
+ cursor.execute(*query)
+ amounts.update(cursor)
for record in records:
amount = amounts[record.id]
@@ -734,18 +732,19 @@ class BudgetLinePeriod(AmountMixin, ModelSQL, ModelView):
transaction = Transaction()
cursor = transaction.connection.cursor()
table = cls.__table__()
- for sub_ids in grouped_slice({p.budget_line.id for p in periods}):
- cursor.execute(*table.select(
- table.budget_line,
- where=fields.SQL_OPERATORS['in'](
- table.budget_line, sub_ids),
- group_by=table.budget_line,
- having=Sum(table.ratio) > 1,
- limit=1))
- try:
- line_id, = cursor.fetchone()
- except TypeError:
- continue
+ line_ids = {p.budget_line.id for p in periods}
+ cursor.execute(*table.select(
+ table.budget_line,
+ where=fields.SQL_OPERATORS['in'](
+ table.budget_line, line_ids),
+ group_by=table.budget_line,
+ having=Sum(table.ratio) > 1,
+ limit=1))
+ try:
+ line_id, = cursor.fetchone()
+ except TypeError:
+ pass
+ else:
line = Line(line_id)
raise BudgetValidationError(
gettext('account_budget.msg_budget_line_period_ratio',
diff --git a/modules/account_deposit/party.py b/modules/account_deposit/party.py
index 2a69430f7b..ffc8c754fe 100644
--- a/modules/account_deposit/party.py
+++ b/modules/account_deposit/party.py
@@ -12,7 +12,7 @@ from trytond.model import fields
from trytond.modules.currency.fields import Monetary
from trytond.modules.party.exceptions import EraseError
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
@@ -45,26 +45,25 @@ class Party(metaclass=PoolMeta):
line_clause, _ = MoveLine.query_get(line)
- for sub_parties in grouped_slice(parties):
- party_clause = fields.SQL_OPERATORS['in'](
- line.party, [p.id for p in sub_parties])
- query = (line
- .join(account, condition=account.id == line.account)
- .join(account_type, condition=account.type == account_type.id)
- .select(line.party,
- # Use credit - debit to positive deposit amount
- Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0)
- ).as_('debit'),
- where=account_type.deposit
- & party_clause
- & (line.reconciliation == Null)
- & (account.company == user.company.id)
- & line_clause,
- group_by=line.party))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- values.update((p, currency.round(a)) for p, a in cursor)
+ party_clause = fields.SQL_OPERATORS['in'](
+ line.party, [p.id for p in parties])
+ query = (line
+ .join(account, condition=account.id == line.account)
+ .join(account_type, condition=account.type == account_type.id)
+ .select(line.party,
+ # Use credit - debit to positive deposit amount
+ Sum(Coalesce(line.credit, 0) - Coalesce(line.debit, 0)
+ ).as_('debit'),
+ where=account_type.deposit
+ & party_clause
+ & (line.reconciliation == Null)
+ & (account.company == user.company.id)
+ & line_clause,
+ group_by=line.party))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ values.update((p, currency.round(a)) for p, a in cursor)
return values
@classmethod
diff --git a/modules/account_dunning_letter/dunning.py b/modules/account_dunning_letter/dunning.py
index fae4ebd35c..5ae15e39ba 100644
--- a/modules/account_dunning_letter/dunning.py
+++ b/modules/account_dunning_letter/dunning.py
@@ -1,12 +1,11 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-from itertools import chain, groupby
+from itertools import groupby
from operator import attrgetter
from trytond.model import fields
from trytond.modules.company import CompanyReport
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from trytond.wizard import StateReport
@@ -120,19 +119,16 @@ class Letter(CompanyReport, metaclass=PoolMeta):
"""
pool = Pool()
Line = pool.get('account.move.line')
- payments = []
- for sub_parties in grouped_slice(parties):
- payments.append(Line.search([
- ('account.type.receivable', '=', True),
- ['OR',
- ('debit', '<', 0),
- ('credit', '>', 0),
- ],
- ('party', 'in', [p.id for p in sub_parties]),
- ('reconciliation', '=', None),
- ],
- order=[('party', 'ASC'), ('id', 'ASC')]))
- payments = list(chain(*payments))
+ payments = Line.search([
+ ('account.type.receivable', '=', True),
+ ['OR',
+ ('debit', '<', 0),
+ ('credit', '>', 0),
+ ],
+ ('party', 'in', parties),
+ ('reconciliation', '=', None),
+ ],
+ order=[('party', 'ASC'), ('id', 'ASC')])
return dict((party, list(payments))
for party, payments in groupby(payments, attrgetter('party')))
diff --git a/modules/account_es_sii/account.py b/modules/account_es_sii/account.py
index 4369058a6a..a77a6e3752 100644
--- a/modules/account_es_sii/account.py
+++ b/modules/account_es_sii/account.py
@@ -245,15 +245,14 @@ class Period(metaclass=PoolMeta):
def check_es_sii_posted_invoices(cls, periods):
pool = Pool()
Invoice = pool.get('account.invoice')
- for sub_ids in grouped_slice(list(map(int, periods))):
- invoices = Invoice.search([
- ('move.period', 'in', sub_ids),
- ], limit=1, order=[])
- if invoices:
- invoice, = invoices
- raise ESSIIPostedInvoicesError(gettext(
- 'account_es_sii.msg_es_sii_posted_invoices',
- period=invoice.move.period.rec_name))
+ invoices = Invoice.search([
+ ('move.period', 'in', periods),
+ ], limit=1, order=[])
+ if invoices:
+ invoice, = invoices
+ raise ESSIIPostedInvoicesError(gettext(
+ 'account_es_sii.msg_es_sii_posted_invoices',
+ period=invoice.move.period.rec_name))
class Invoice(metaclass=PoolMeta):
diff --git a/modules/account_invoice/account.py b/modules/account_invoice/account.py
index 661f30e17a..8b235bfa5a 100644
--- a/modules/account_invoice/account.py
+++ b/modules/account_invoice/account.py
@@ -1,6 +1,6 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
from itertools import islice
from trytond.i18n import gettext
@@ -10,7 +10,6 @@ from trytond.modules.account.exceptions import ClosePeriodError
from trytond.modules.company.model import CompanyValueMixin
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, Id, If
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .exceptions import CancelInvoiceMoveWarning
@@ -224,13 +223,11 @@ class MoveLine(metaclass=PoolMeta):
pool = Pool()
InvoicePaymentLine = pool.get('account.invoice-account.move.line')
- ids = list(map(int, lines))
- result = dict.fromkeys(ids, None)
- for sub_ids in grouped_slice(ids):
- payment_lines = InvoicePaymentLine.search([
- ('line', 'in', list(sub_ids)),
- ])
- result.update({p.line.id: p.invoice.id for p in payment_lines})
+ result = defaultdict(lambda: None)
+ payment_lines = InvoicePaymentLine.search([
+ ('line', 'in', lines),
+ ])
+ result.update({p.line.id: p.invoice.id for p in payment_lines})
return result
@classmethod
@@ -269,13 +266,10 @@ def _invoices_to_process(reconciliations):
move_ids.add(line.move.id)
others.update(line.reconciliations_delegated)
- invoices = set()
- for sub_ids in grouped_slice(move_ids):
- sub_ids = list(sub_ids)
- invoices.update(Invoice.search(['OR',
- ('move', 'in', sub_ids),
- ('additional_moves', 'in', sub_ids),
- ]))
+ invoices = set(Invoice.search(['OR',
+ ('move', 'in', move_ids),
+ ('additional_moves', 'in', move_ids),
+ ]))
if others:
invoices.update(_invoices_to_process(Reconciliation.browse(others)))
diff --git a/modules/account_invoice/invoice.py b/modules/account_invoice/invoice.py
index 31169292ca..9736cb5984 100644
--- a/modules/account_invoice/invoice.py
+++ b/modules/account_invoice/invoice.py
@@ -828,17 +828,16 @@ class Invoice(
type_name = cls.tax_amount._field.sql_type().base
tax = InvoiceTax.__table__()
- for sub_invoices in grouped_slice(invoices_no_cache):
- red_sql = fields.SQL_OPERATORS['in'](
- tax.invoice, map(int, sub_invoices))
- query = (tax.select(tax.invoice,
- Coalesce(Sum(tax.amount), 0).as_(type_name),
- where=red_sql,
- group_by=tax.invoice))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- tax_amount.update(cursor)
+ where = fields.SQL_OPERATORS['in'](
+ tax.invoice, map(int, invoices_no_cache))
+ query = (tax.select(tax.invoice,
+ Coalesce(Sum(tax.amount), 0).as_(type_name),
+ where=where,
+ group_by=tax.invoice))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ tax_amount.update(cursor)
# Float amount must be rounded to get the right precision
if backend.name == 'sqlite':
for invoice in invoices:
@@ -915,14 +914,13 @@ class Invoice(
def get_lines_to_pay(cls, invoices, name):
cursor = Transaction().connection.cursor()
lines = defaultdict(list)
- for sub_invoices in grouped_slice(invoices):
- query = cls._query_lines_to_pay(sub_invoices)
- query = query.select(
- query.invoice, query.line,
- order_by=query.maturity_date.nulls_last)
- cursor.execute(*query)
- for invoice_id, line_id in cursor:
- lines[invoice_id].append(line_id)
+ query = cls._query_lines_to_pay(invoices)
+ query = query.select(
+ query.invoice, query.line,
+ order_by=query.maturity_date.nulls_last)
+ cursor.execute(*query)
+ for invoice_id, line_id in cursor:
+ lines[invoice_id].append(line_id)
return lines
@classmethod
@@ -934,27 +932,25 @@ class Invoice(
move = Move.__table__()
cursor = Transaction().connection.cursor()
lines = defaultdict(list)
- for sub_invoices in grouped_slice(invoices):
- sub_invoices = list(sub_invoices)
- lines_to_pay = cls._query_lines_to_pay(sub_invoices)
- query = cls._query_lines_to_pay(sub_invoices)
- query = (
- query
- .join(line,
- condition=query.reconciliation == line.reconciliation)
- .join(move, condition=line.move == move.id)
- .select(
- query.invoice, line.id,
- where=~Exists(
- lines_to_pay.select(
- lines_to_pay.line,
- where=(lines_to_pay.invoice == query.invoice)
- & (lines_to_pay.line == line.id))),
- group_by=[query.invoice, line.id, move.date],
- order_by=move.date.asc))
- cursor.execute(*query)
- for invoice_id, line_id in cursor:
- lines[invoice_id].append(line_id)
+ lines_to_pay = cls._query_lines_to_pay(invoices)
+ query = cls._query_lines_to_pay(invoices)
+ query = (
+ query
+ .join(line,
+ condition=query.reconciliation == line.reconciliation)
+ .join(move, condition=line.move == move.id)
+ .select(
+ query.invoice, line.id,
+ where=~Exists(
+ lines_to_pay.select(
+ lines_to_pay.line,
+ where=(lines_to_pay.invoice == query.invoice)
+ & (lines_to_pay.line == line.id))),
+ group_by=[query.invoice, line.id, move.date],
+ order_by=move.date.asc))
+ cursor.execute(*query)
+ for invoice_id, line_id in cursor:
+ lines[invoice_id].append(line_id)
return lines
@classmethod
@@ -1146,16 +1142,13 @@ class Invoice(
table = cls.__table__()
cursor = Transaction().connection.cursor()
- result = {}
has_cache = (
(table.invoice_report_cache_id != Null)
| (table.invoice_report_cache != Null))
- for sub_invoices in grouped_slice(invoices):
- sub_ids = map(int, sub_invoices)
- cursor.execute(*table.select(table.id, has_cache,
- where=fields.SQL_OPERATORS['in'](table.id, sub_ids)))
- result.update(cursor)
- return result
+ cursor.execute(*table.select(table.id, has_cache,
+ where=fields.SQL_OPERATORS['in'](
+ table.id, map(int, invoices))))
+ return dict(cursor)
@property
@fields.depends('lines', 'type')
@@ -1896,13 +1889,11 @@ class Invoice(
PaymentLine = pool.get('account.invoice-account.move.line')
payments = defaultdict(list)
- ids = list(map(int, lines))
- for sub_ids in grouped_slice(ids):
- payment_lines = PaymentLine.search([
- ('line', 'in', list(sub_ids)),
- ])
- for payment_line in payment_lines:
- payments[payment_line.invoice].append(payment_line.line)
+ payment_lines = PaymentLine.search([
+ ('line', 'in', lines),
+ ])
+ for payment_line in payment_lines:
+ payments[payment_line.invoice].append(payment_line.line)
to_write = []
for invoice, lines in payments.items():
@@ -2144,7 +2135,8 @@ class Invoice(
def _check_similar(cls, invoices, type='in'):
pool = Pool()
Warning = pool.get('res.user.warning')
- for sub_invoices in grouped_slice(invoices):
+ for sub_invoices in grouped_slice(
+ invoices, backend.MAX_QUERY_PARAMS // 10):
sub_invoices = list(sub_invoices)
domain = list(filter(None,
(i._similar_domain() for i in sub_invoices
diff --git a/modules/account_invoice_history/account.py b/modules/account_invoice_history/account.py
index ad26dcd29c..630e46205f 100644
--- a/modules/account_invoice_history/account.py
+++ b/modules/account_invoice_history/account.py
@@ -1,5 +1,8 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
+
+from collections import defaultdict
+
from sql import Null
from sql.conditionals import Greatest
from sql.functions import CurrentTimestamp
@@ -7,7 +10,7 @@ from sql.functions import CurrentTimestamp
from trytond import backend
from trytond.model import ModelView, Workflow, fields
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
@@ -41,27 +44,25 @@ class Invoice(metaclass=PoolMeta):
payment_term = PaymentTerm.__table__()
cursor = Transaction().connection.cursor()
- invoice_ids = [i.id for i in invoices]
- datetimes = dict.fromkeys(invoice_ids)
- for ids in grouped_slice(invoice_ids):
- query = (table
- .join(party, condition=table.party == party.id)
- .join(address, condition=table.invoice_address == address.id)
- .join(identifier, 'LEFT',
- condition=table.party_tax_identifier == identifier.id)
- .join(payment_term, 'LEFT',
- condition=table.payment_term == payment_term.id)
- .select(table.id,
- Greatest(table.numbered_at, party.create_date,
- address.create_date, identifier.create_date,
- payment_term.create_date).as_('history_datetime'),
- where=fields.SQL_OPERATORS['in'](table.id, ids)
- & (table.numbered_at != Null)
- & (table.state.in_(cls._history_states()))))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'DATETIME'])
- cursor.execute(*query)
- datetimes.update(cursor)
+ datetimes = defaultdict(lambda: None)
+ query = (table
+ .join(party, condition=table.party == party.id)
+ .join(address, condition=table.invoice_address == address.id)
+ .join(identifier, 'LEFT',
+ condition=table.party_tax_identifier == identifier.id)
+ .join(payment_term, 'LEFT',
+ condition=table.payment_term == payment_term.id)
+ .select(table.id,
+ Greatest(table.numbered_at, party.create_date,
+ address.create_date, identifier.create_date,
+ payment_term.create_date).as_('history_datetime'),
+ where=fields.SQL_OPERATORS['in'](table.id, map(int, invoices))
+ & (table.numbered_at != Null)
+ & (table.state.in_(cls._history_states()))))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'DATETIME'])
+ cursor.execute(*query)
+ datetimes.update(cursor)
return datetimes
@classmethod
diff --git a/modules/account_payment/account.py b/modules/account_payment/account.py
index 5c5078ca0e..47fc864c0c 100644
--- a/modules/account_payment/account.py
+++ b/modules/account_payment/account.py
@@ -19,7 +19,6 @@ from trytond.modules.company.model import CompanyValueMixin
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, Id, If
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction, check_access, without_check_access
from trytond.wizard import (
Button, StateAction, StateTransition, StateView, Wizard)
@@ -192,12 +191,9 @@ class MoveLine(metaclass=PoolMeta):
where=table.account.in_(accounts)))
if lines:
- for sub_lines in grouped_slice(lines):
- query.where = (
- table.account.in_(accounts)
- & fields.SQL_OPERATORS['in'](
- table.id, map(int, sub_lines)))
- cursor.execute(*query)
+ query.where &= fields.SQL_OPERATORS['in'](
+ table.id, map(int, lines))
+ cursor.execute(*query)
else:
cursor.execute(*query)
diff --git a/modules/account_payment/payment.py b/modules/account_payment/payment.py
index f3b32db5c2..2577397639 100644
--- a/modules/account_payment/payment.py
+++ b/modules/account_payment/payment.py
@@ -21,8 +21,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval, If
from trytond.rpc import RPC
-from trytond.tools import (
- cursor_dict, grouped_slice, sortable_values, sqlite_apply_types)
+from trytond.tools import cursor_dict, sortable_values, sqlite_apply_types
from trytond.transaction import Transaction
from trytond.wizard import StateAction, Wizard
@@ -201,32 +200,31 @@ class Group(ModelSQL, ModelView, ChatMixin):
).as_('payment_not_complete'),
]
- for sub_groups in grouped_slice(groups):
- query = payment.select(*columns,
- where=fields.SQL_OPERATORS['in'](
- payment.group, map(int, sub_groups)),
- group_by=payment.group)
- if backend.name == 'sqlite':
- sqlite_apply_types(
- query, [None, None, 'NUMERIC', 'NUMERIC', None])
- cursor.execute(*query)
- for row in cursor_dict(cursor):
- group = cls(row['group_id'])
+ query = payment.select(*columns,
+ where=fields.SQL_OPERATORS['in'](
+ payment.group, map(int, groups)),
+ group_by=payment.group)
+ if backend.name == 'sqlite':
+ sqlite_apply_types(
+ query, [None, None, 'NUMERIC', 'NUMERIC', None])
+ cursor.execute(*query)
+ for row in cursor_dict(cursor):
+ group = cls(row['group_id'])
- result['payment_count'][group.id] = row['payment_count']
- result['payment_complete'][group.id] = \
- not row['payment_not_complete']
+ result['payment_count'][group.id] = row['payment_count']
+ result['payment_complete'][group.id] = \
+ not row['payment_not_complete']
- amount = row['payment_amount']
- succeeded = row['payment_amount_succeeded']
+ amount = row['payment_amount']
+ succeeded = row['payment_amount_succeeded']
- if amount is not None and backend.name == 'sqlite':
- amount = group.company.currency.round(amount)
- result['payment_amount'][group.id] = amount
+ if amount is not None and backend.name == 'sqlite':
+ amount = group.company.currency.round(amount)
+ result['payment_amount'][group.id] = amount
- if succeeded is not None and backend.name == 'sqlite':
- succeeded = group.company.currency.round(succeeded)
- result['payment_amount_succeeded'][group.id] = succeeded
+ if succeeded is not None and backend.name == 'sqlite':
+ succeeded = group.company.currency.round(succeeded)
+ result['payment_amount_succeeded'][group.id] = succeeded
for key in list(result.keys()):
if key not in names:
diff --git a/modules/account_payment_clearing/account.py b/modules/account_payment_clearing/account.py
index 65d8b23230..6ef69aefba 100644
--- a/modules/account_payment_clearing/account.py
+++ b/modules/account_payment_clearing/account.py
@@ -1,7 +1,6 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
class Move(metaclass=PoolMeta):
@@ -25,11 +24,9 @@ def _payments_to_update(reconciliations):
moves.add(line.move)
others.update(line.reconciliations_delegated)
- payments = set()
- for sub_moves in grouped_slice(moves):
- payments.update(Payment.search([
- ('clearing_move', 'in', [m.id for m in sub_moves]),
- ], order=[]))
+ payments = set(Payment.search([
+ ('clearing_move', 'in', moves),
+ ], order=[]))
if others:
payments.update(_payments_to_update(Reconciliation.browse(others)))
diff --git a/modules/account_payment_clearing/payment.py b/modules/account_payment_clearing/payment.py
index d21c5323da..274bacfb81 100644
--- a/modules/account_payment_clearing/payment.py
+++ b/modules/account_payment_clearing/payment.py
@@ -13,7 +13,6 @@ from trytond.model import ModelView, Workflow, fields
from trytond.modules.account.exceptions import AccountMissing
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, If, TimeDelta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from trytond.wizard import Button, StateTransition, StateView, Wizard
@@ -444,13 +443,12 @@ class Group(metaclass=PoolMeta):
column = Min(column)
else:
column = BoolAnd(column)
- for sub_groups in grouped_slice(groups):
- cursor.execute(*payment.select(
- payment.group, column,
- where=fields.SQL_OPERATORS['in'](
- payment.group, map(int, sub_groups)),
- group_by=payment.group))
- result.update(cursor)
+ cursor.execute(*payment.select(
+ payment.group, column,
+ where=fields.SQL_OPERATORS['in'](
+ payment.group, map(int, groups)),
+ group_by=payment.group))
+ result.update(cursor)
return result
@classmethod
diff --git a/modules/account_payment_sepa/payment.py b/modules/account_payment_sepa/payment.py
index dcaa0513a4..8365a117a2 100644
--- a/modules/account_payment_sepa/payment.py
+++ b/modules/account_payment_sepa/payment.py
@@ -4,6 +4,7 @@ import datetime
import os
import unicodedata
import uuid
+from collections import defaultdict
from io import BytesIO
from itertools import groupby
from operator import attrgetter
@@ -26,8 +27,7 @@ from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval, If
from trytond.report import Report
from trytond.tools import (
- cached_property, grouped_slice, is_full_text, lstrip_wildcard,
- sortable_values)
+ cached_property, is_full_text, lstrip_wildcard, sortable_values)
from trytond.transaction import Transaction
from .sepa_handler import CAMT054
@@ -785,14 +785,12 @@ class Mandate(Workflow, ModelSQL, ModelView):
payment = Payment.__table__()
cursor = Transaction().connection.cursor()
- has_payments = dict.fromkeys([m.id for m in mandates], False)
- for sub_mandates in grouped_slice(mandates):
- red_sql = fields.SQL_OPERATORS['in'](
- payment.sepa_mandate, map(int, sub_mandates))
- cursor.execute(*payment.select(payment.sepa_mandate, Literal(True),
- where=red_sql,
- group_by=payment.sepa_mandate))
- has_payments.update(cursor)
+ has_payments = defaultdict(lambda: False)
+ cursor.execute(*payment.select(payment.sepa_mandate, Literal(True),
+ where=fields.SQL_OPERATORS['in'](
+ payment.sepa_mandate, map(int, mandates)),
+ group_by=payment.sepa_mandate))
+ has_payments.update(cursor)
return has_payments
@@ -803,22 +801,20 @@ class Mandate(Workflow, ModelSQL, ModelView):
payment = Payment.__table__()
cursor = Transaction().connection.cursor()
- is_first = dict.fromkeys([m.id for m in mandates], True)
- for sub_mandates in grouped_slice(mandates):
- red_sql = fields.SQL_OPERATORS['in'](
- payment.sepa_mandate, map(int, sub_mandates))
- cursor.execute(*payment.select(
- payment.sepa_mandate, Literal(False),
- where=red_sql
- & (payment.sepa_mandate_sequence_type != Null)
- & ~( # Same as property rejected
- (payment.state == 'failed')
- & ((payment.sepa_return_reason_code != Null)
- | (payment.sepa_return_reason_code != ''))
- & (payment.sepa_return_reason_information
- == '/RTYP/RJCT')),
- group_by=payment.sepa_mandate))
- is_first.update(cursor)
+ is_first = defaultdict(lambda: True)
+ cursor.execute(*payment.select(
+ payment.sepa_mandate, Literal(False),
+ where=fields.SQL_OPERATORS['in'](
+ payment.sepa_mandate, map(int, mandates))
+ & (payment.sepa_mandate_sequence_type != Null)
+ & ~( # Same as property rejected
+ (payment.state == 'failed')
+ & ((payment.sepa_return_reason_code != Null)
+ | (payment.sepa_return_reason_code != ''))
+ & (payment.sepa_return_reason_information
+ == '/RTYP/RJCT')),
+ group_by=payment.sepa_mandate))
+ is_first.update(cursor)
return is_first
@classmethod
diff --git a/modules/account_statement/bank.py b/modules/account_statement/bank.py
index 1f6841eabe..bfb4643bc6 100644
--- a/modules/account_statement/bank.py
+++ b/modules/account_statement/bank.py
@@ -4,7 +4,6 @@
from trytond.i18n import gettext
from trytond.modules.bank.exceptions import AccountValidationError
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
class Account(metaclass=PoolMeta):
@@ -21,16 +20,14 @@ class Account(metaclass=PoolMeta):
Journal = pool.get('account.statement.journal')
if field_names and 'currency' not in field_names:
return
- for sub_accounts in grouped_slice(accounts):
- sub_account_ids = [a.id for a in sub_accounts]
- journals = Journal.search([
- ('bank_account', 'in', sub_account_ids),
- ])
- for journal in journals:
- if (journal.currency != journal.bank_account.currency
- and journal.bank_account.currency):
- raise AccountValidationError(
- gettext('account_statement.msg_bank_account_currency',
- bank_account=journal.bank_account.rec_name,
- currency=journal.currency.rec_name,
- journal=journal.rec_name))
+ journals = Journal.search([
+ ('bank_account', 'in', accounts),
+ ])
+ for journal in journals:
+ if (journal.currency != journal.bank_account.currency
+ and journal.bank_account.currency):
+ raise AccountValidationError(
+ gettext('account_statement.msg_bank_account_currency',
+ bank_account=journal.bank_account.rec_name,
+ currency=journal.currency.rec_name,
+ journal=journal.rec_name))
diff --git a/modules/account_statement/journal.py b/modules/account_statement/journal.py
index 8dec0a7cc7..fe7b0e9ade 100644
--- a/modules/account_statement/journal.py
+++ b/modules/account_statement/journal.py
@@ -1,6 +1,8 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
+from collections import defaultdict
+
from sql import Window
from sql.functions import FirstValue
@@ -10,7 +12,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Eval
from trytond.rpc import RPC
-from trytond.tools import cursor_dict, grouped_slice, sqlite_apply_types
+from trytond.tools import cursor_dict, sqlite_apply_types
from trytond.transaction import Transaction
@@ -129,38 +131,37 @@ class Journal(DeactivableMixin, ModelSQL, ModelView):
result = {}
for name in names:
- result[name] = dict.fromkeys(id2currency.keys(), None)
+ result[name] = defaultdict(lambda: None)
- for sub_ids in grouped_slice(id2currency.keys()):
- columns = [statement.journal]
- w = Window(
- partition=[statement.journal],
- order_by=[statement.date.desc, statement.id.desc])
+ columns = [statement.journal]
+ w = Window(
+ partition=[statement.journal],
+ order_by=[statement.date.desc, statement.id.desc])
+ if 'last_amount' in names:
+ columns.append(
+ FirstValue(statement.end_balance,
+ window=w).as_('last_amount'))
+ if 'last_date' in names:
+ columns.append(
+ FirstValue(statement.date, window=w).as_('last_date'))
+ query = statement.select(*columns,
+ distinct=True,
+ where=(fields.SQL_OPERATORS['in'](
+ statement.journal, id2currency.keys())
+ & (statement.state != 'cancelled')))
+ if backend.name == 'sqlite':
+ types = [None]
if 'last_amount' in names:
- columns.append(
- FirstValue(statement.end_balance,
- window=w).as_('last_amount'))
+ types.append('NUMERIC')
if 'last_date' in names:
- columns.append(
- FirstValue(statement.date, window=w).as_('last_date'))
- query = statement.select(*columns,
- distinct=True,
- where=(fields.SQL_OPERATORS['in'](
- statement.journal, sub_ids)
- & (statement.state != 'cancelled')))
- if backend.name == 'sqlite':
- types = [None]
- if 'last_amount' in names:
- types.append('NUMERIC')
- if 'last_date' in names:
- types.append('DATE')
- sqlite_apply_types(query, types)
- cursor.execute(*query)
- for row in cursor_dict(cursor):
- journal = row['journal']
- if 'last_amount' in names:
- result['last_amount'][journal] = (
- id2currency[journal].round(row['last_amount']))
- if 'last_date' in names:
- result['last_date'][journal] = row['last_date']
+ types.append('DATE')
+ sqlite_apply_types(query, types)
+ cursor.execute(*query)
+ for row in cursor_dict(cursor):
+ journal = row['journal']
+ if 'last_amount' in names:
+ result['last_amount'][journal] = (
+ id2currency[journal].round(row['last_amount']))
+ if 'last_date' in names:
+ result['last_date'][journal] = row['last_date']
return result
diff --git a/modules/analytic_account/account.py b/modules/analytic_account/account.py
index 29b7cbb8c8..ff82b58ca8 100644
--- a/modules/analytic_account/account.py
+++ b/modules/analytic_account/account.py
@@ -16,8 +16,7 @@ from trytond.model.exceptions import AccessError
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Eval, If, PYSONDecoder, PYSONEncoder
-from trytond.tools import (
- grouped_slice, is_full_text, lstrip_wildcard, sqlite_apply_types)
+from trytond.tools import is_full_text, lstrip_wildcard, sqlite_apply_types
from trytond.transaction import Transaction
from .exceptions import AccountValidationError
@@ -140,16 +139,14 @@ class Account(
return
accounts = [
a for a in accounts if a.type in {'root', 'view', 'distribution'}]
- for sub_accounts in grouped_slice(accounts):
- sub_accounts = list(sub_accounts)
- lines = Line.search([
- ('account', 'in', [a.id for a in sub_accounts]),
- ], order=[], limit=1)
- if lines:
- line, = lines
- raise AccountValidationError(gettext(
- 'analytic_account.msg_account_wrong_type_line',
- account=line.account.rec_name))
+ lines = Line.search([
+ ('account', 'in', accounts),
+ ], order=[], limit=1)
+ if lines:
+ line, = lines
+ raise AccountValidationError(gettext(
+ 'analytic_account.msg_account_wrong_type_line',
+ account=line.account.rec_name))
@fields.depends('company')
def on_change_with_currency(self, name=None):
@@ -313,17 +310,16 @@ class Account(
mode, accounts, values=values, external=external)
if mode == 'write' and 'root' in values:
- for sub_records in grouped_slice(accounts):
- entries = Entry.search([
- ('account', 'in', list(map(int, sub_records))),
- ],
- limit=1, order=[])
- if entries:
- entry, = entries
- raise AccessError(gettext(
- 'analytic_account'
- '.msg_analytic_account_root_change',
- account=entry.account.rec_name))
+ entries = Entry.search([
+ ('account', 'in', accounts),
+ ],
+ limit=1, order=[])
+ if entries:
+ entry, = entries
+ raise AccessError(gettext(
+ 'analytic_account'
+ '.msg_analytic_account_root_change',
+ account=entry.account.rec_name))
class AccountContext(ModelView):
diff --git a/modules/commission/commission.py b/modules/commission/commission.py
index f4956e3fff..7e24456561 100644
--- a/modules/commission/commission.py
+++ b/modules/commission/commission.py
@@ -1,6 +1,7 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
+from collections import defaultdict
from decimal import Decimal
from itertools import groupby
@@ -21,7 +22,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, Id, If
-from trytond.tools import decistmt, grouped_slice, sqlite_apply_types
+from trytond.tools import decistmt, sqlite_apply_types
from trytond.transaction import Transaction, check_access
from trytond.wizard import Button, StateAction, StateView, Wizard
@@ -125,19 +126,17 @@ class Agent(DeactivableMixin, ModelSQL, ModelView):
commission = Commission.__table__()
cursor = Transaction().connection.cursor()
- ids = [a.id for a in agents]
- amounts = dict.fromkeys(ids, None)
- for sub_ids in grouped_slice(ids):
- where = fields.SQL_OPERATORS['in'](commission.agent, sub_ids)
- where &= commission.invoice_line == Null
- query = commission.select(
- commission.agent, Sum(commission.amount).as_('pending_amount'),
- where=where,
- group_by=commission.agent)
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- amounts.update(dict(cursor))
+ amounts = defaultdict(lambda: None)
+ query = commission.select(
+ commission.agent, Sum(commission.amount).as_('pending_amount'),
+ where=fields.SQL_OPERATORS['in'](
+ commission.agent, map(int, agents))
+ & (commission.invoice_line == Null),
+ group_by=commission.agent)
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ amounts.update(dict(cursor))
if backend.name == 'sqlite':
for agent_id, amount in amounts.items():
if amount is not None:
diff --git a/modules/commission/invoice.py b/modules/commission/invoice.py
index 79f83a7826..2dd35e55a1 100644
--- a/modules/commission/invoice.py
+++ b/modules/commission/invoice.py
@@ -7,7 +7,6 @@ from trytond.model import ModelView, Workflow, fields
from trytond.modules.product import round_price
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -59,15 +58,13 @@ class Invoice(metaclass=PoolMeta):
for company, c_invoices in groupby(invoices, key=lambda i: i.company):
with Transaction().set_context(company=company.id):
today = Date.today()
- for sub_invoices in grouped_slice(list(c_invoices)):
- ids = [i.id for i in sub_invoices]
- for commission in Commission.search([
- ('date', '=', None),
- ('origin.invoice', 'in', ids,
- 'account.invoice.line'),
- ]):
- date = commission.origin.invoice.reconciled or today
- date2commissions[date].append(commission)
+ for commission in Commission.search([
+ ('date', '=', None),
+ ('origin.invoice', 'in', list(c_invoices),
+ 'account.invoice.line'),
+ ]):
+ date = commission.origin.invoice.reconciled or today
+ date2commissions[date].append(commission)
to_write = []
for date, commissions in date2commissions.items():
to_write.append(commissions)
@@ -103,19 +100,19 @@ class Invoice(metaclass=PoolMeta):
to_delete = []
to_save = []
- for sub_invoices in grouped_slice(invoices_to_revert_commission):
- ids = [i.id for i in sub_invoices]
- to_delete += Commission.search([
- ('invoice_line', '=', None),
- ('origin.invoice', 'in', ids, 'account.invoice.line'),
- ])
- to_cancel = Commission.search([
- ('invoice_line', '!=', None),
- ('origin.invoice', 'in', ids, 'account.invoice.line'),
- ])
- for commission in Commission.copy(to_cancel):
- commission.amount *= -1
- to_save.append(commission)
+ to_delete += Commission.search([
+ ('invoice_line', '=', None),
+ ('origin.invoice', 'in', invoices_to_revert_commission,
+ 'account.invoice.line'),
+ ])
+ to_cancel = Commission.search([
+ ('invoice_line', '!=', None),
+ ('origin.invoice', 'in', invoices_to_revert_commission,
+ 'account.invoice.line'),
+ ])
+ for commission in Commission.copy(to_cancel):
+ commission.amount *= -1
+ to_save.append(commission)
Commission.delete(to_delete)
Commission.save(to_save)
diff --git a/modules/customs/product.py b/modules/customs/product.py
index 95a651fc35..86fecad994 100644
--- a/modules/customs/product.py
+++ b/modules/customs/product.py
@@ -3,7 +3,6 @@
from trytond.model import ModelSQL, ModelView, fields, sequence_ordered
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, Or
-from trytond.tools import grouped_slice
class Category(metaclass=PoolMeta):
@@ -82,15 +81,10 @@ class Category(metaclass=PoolMeta):
callback = super().on_delete(categories)
- product_tariffcodes = set()
- products = [str(t) for t in categories]
- for products in grouped_slice(products):
- product_tariffcodes.update(Product_TariffCode.search([
- 'product', 'in', list(products),
- ]))
+ product_tariffcodes = Product_TariffCode.search([
+ 'product', 'in', [str(t) for t in categories],
+ ])
if product_tariffcodes:
- product_tariffcodes = Product_TariffCode.browse(
- product_tariffcodes)
callback.append(
lambda: Product_TariffCode.delete(product_tariffcodes))
return callback
@@ -157,15 +151,10 @@ class Template(metaclass=PoolMeta):
callback = super().on_delete(templates)
- product_tariffcodes = set()
- products = [str(t) for t in templates]
- for products in grouped_slice(products):
- product_tariffcodes.update(Product_TariffCode.search([
- 'product', 'in', list(products),
- ]))
+ product_tariffcodes = Product_TariffCode.search([
+ 'product', 'in', [str(t) for t in templates],
+ ])
if product_tariffcodes:
- product_tariffcodes = Product_TariffCode.browse(
- product_tariffcodes)
callback.append(
lambda: Product_TariffCode.delete(product_tariffcodes))
return callback
diff --git a/modules/marketing_automation/marketing_automation.py b/modules/marketing_automation/marketing_automation.py
index d8a40eccbc..a8ae786bf9 100644
--- a/modules/marketing_automation/marketing_automation.py
+++ b/modules/marketing_automation/marketing_automation.py
@@ -26,7 +26,7 @@ from trytond.pool import Pool
from trytond.pyson import Eval, If, PYSONDecoder, TimeDelta
from trytond.report import Report, html_to_text, mjml_to_html
from trytond.sendmail import SMTPDataManager, send_message_transactional
-from trytond.tools import grouped_slice, pairwise_longest
+from trytond.tools import pairwise_longest
from trytond.tools.chart import sparkline
from trytond.tools.email_ import format_address, has_rcpt, set_from_header
from trytond.transaction import Transaction
@@ -178,19 +178,18 @@ class Scenario(
others.append(scenario)
count = {name: defaultdict(int) for name in names}
- for sub_others in grouped_slice(others):
- cursor.execute(*record.select(
- record.scenario,
- Count(),
- Count(filter_=record.blocked),
- where=fields.SQL_OPERATORS['in'](
- record.scenario, map(int, sub_others)),
- group_by=record.scenario))
- for id_, all_, blocked in cursor:
- if 'record_count' in count:
- count['record_count'][id_] = all_
- if 'record_count_blocked' in count:
- count['record_count_blocked'][id_] = blocked
+ cursor.execute(*record.select(
+ record.scenario,
+ Count(),
+ Count(filter_=record.blocked),
+ where=fields.SQL_OPERATORS['in'](
+ record.scenario, map(int, others)),
+ group_by=record.scenario))
+ for id_, all_, blocked in cursor:
+ if 'record_count' in count:
+ count['record_count'][id_] = all_
+ if 'record_count_blocked' in count:
+ count['record_count_blocked'][id_] = blocked
for scenario in drafts:
Model = pool.get(scenario.model)
domain = PYSONDecoder({}).decode(scenario.domain)
@@ -522,22 +521,21 @@ class Activity(
cursor = Transaction().connection.cursor()
count = {name: defaultdict(int) for name in names}
- for sub_activities in grouped_slice(activities):
- cursor.execute(*record_activity.select(
- record_activity.activity,
- Count(filter_=record_activity.state == 'done'),
- Count(filter_=record_activity.email_opened),
- Count(filter_=record_activity.email_clicked),
- where=fields.SQL_OPERATORS['in'](
- record_activity.activity, map(int, sub_activities)),
- group_by=record_activity.activity))
- for id_, all_, email_opened, email_clicked in cursor:
- if 'record_count' in count:
- count['record_count'][id_] = all_
- if 'email_opened' in count:
- count['email_opened'][id_] = email_opened
- if 'email_clicked' in count:
- count['email_clicked'][id_] = email_clicked
+ cursor.execute(*record_activity.select(
+ record_activity.activity,
+ Count(filter_=record_activity.state == 'done'),
+ Count(filter_=record_activity.email_opened),
+ Count(filter_=record_activity.email_clicked),
+ where=fields.SQL_OPERATORS['in'](
+ record_activity.activity, map(int, activities)),
+ group_by=record_activity.activity))
+ for id_, all_, email_opened, email_clicked in cursor:
+ if 'record_count' in count:
+ count['record_count'][id_] = all_
+ if 'email_opened' in count:
+ count['email_opened'][id_] = email_opened
+ if 'email_clicked' in count:
+ count['email_clicked'][id_] = email_clicked
return count
@classmethod
diff --git a/modules/marketing_email/marketing.py b/modules/marketing_email/marketing.py
index 210673d159..7b65f9ac92 100644
--- a/modules/marketing_email/marketing.py
+++ b/modules/marketing_email/marketing.py
@@ -22,7 +22,6 @@ from trytond.pool import Pool
from trytond.pyson import Eval
from trytond.report import Report, get_email, html_to_text, mjml_to_html
from trytond.sendmail import SMTPDataManager, send_message_transactional
-from trytond.tools import grouped_slice
from trytond.tools.email_ import (
EmailNotValidError, format_address, normalize_email, set_from_header,
validate_email)
@@ -84,23 +83,22 @@ class Email(DeactivableMixin, ModelSQL, ModelView):
result = {}
web_user = 'web_user' in names
if web_user:
- web_users = dict.fromkeys(list(map(int, records)))
+ web_users = defaultdict(lambda: None)
result['web_user'] = web_users
party = 'party' in names
if party:
- parties = dict.fromkeys(list(map(int, records)))
+ parties = defaultdict(lambda: None)
result['party'] = parties
- for sub_records in grouped_slice(records):
- email2id = {r.email: r.id for r in sub_records}
- users = WebUser.search([
- ('email', 'in', list(email2id.keys())),
- ])
- if web_user:
- web_users.update((email2id[u.email], u.id) for u in users)
- if party:
- parties.update(
- (email2id[u.email], u.party.id)
- for u in users if u.party)
+ email2id = {r.email: r.id for r in records}
+ users = WebUser.search([
+ ('email', 'in', list(email2id.keys())),
+ ])
+ if web_user:
+ web_users.update((email2id[u.email], u.id) for u in users)
+ if party:
+ parties.update(
+ (email2id[u.email], u.party.id)
+ for u in users if u.party)
return result
@classmethod
@@ -256,14 +254,14 @@ class EmailList(DeactivableMixin, ModelSQL, ModelView):
subscribed = defaultdict(int)
query = email.select(
- email.list_, Count(), group_by=[email.list_])
- for sub_lists in grouped_slice(lists):
- query.where = (
+ email.list_, Count(),
+ where=(
fields.SQL_OPERATORS['in'](
- email.list_, map(int, sub_lists))
- & email.active)
- cursor.execute(*query)
- subscribed.update(cursor)
+ email.list_, map(int, lists))
+ & email.active),
+ group_by=[email.list_])
+ cursor.execute(*query)
+ subscribed.update(cursor)
return subscribed
def request_subscribe(self, email, from_=None):
diff --git a/modules/product_price_list_cache/product.py b/modules/product_price_list_cache/product.py
index 93c47c2523..83b79a5f61 100644
--- a/modules/product_price_list_cache/product.py
+++ b/modules/product_price_list_cache/product.py
@@ -10,7 +10,6 @@ from trytond.cache import Cache, freeze
from trytond.model import ModelSQL, dualmethod, fields
from trytond.pool import Pool, PoolMeta
from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
dumps = partial(
@@ -163,23 +162,17 @@ class PriceListCache(ModelSQL):
if price_lists is None and products is None:
cursor.execute(*cache.delete())
elif price_lists and products is None:
- for sub_price_lists in grouped_slice(price_lists):
- cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in'](
- cache.price_list, [
- p.id for p in sub_price_lists])))
+ cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in'](
+ cache.price_list, map(int, price_lists))))
elif price_lists is None and products:
- for sub_products in grouped_slice(products):
- cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in'](
- cache.product, [p.id for p in sub_products])))
+ cursor.execute(*cache.delete(where=fields.SQL_OPERATORS['in'](
+ cache.product, map(int, products))))
else:
- for sub_products in grouped_slice(products):
- for sub_price_lists in grouped_slice(price_lists):
- cursor.execute(*cache.delete(
- where=fields.SQL_OPERATORS['in'](
- cache.price_list, [
- p.id for p in sub_price_lists])
- & fields.SQL_OPERATORS['in'](
- cache.product, [p.id for p in sub_products])))
+ cursor.execute(*cache.delete(
+ where=fields.SQL_OPERATORS['in'](
+ cache.price_list, map(int, price_lists))
+ & fields.SQL_OPERATORS['in'](
+ cache.product, map(int, products))))
@classmethod
def on_modification(cls, mode, records, field_names=None):
diff --git a/modules/production_work/work.py b/modules/production_work/work.py
index 0207d0d08b..2496d5529f 100644
--- a/modules/production_work/work.py
+++ b/modules/production_work/work.py
@@ -19,7 +19,7 @@ from trytond.modules.company.model import employee_field, set_employee
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If, TimeDelta
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
from .exceptions import PickerError
@@ -249,19 +249,16 @@ class Work(sequence_ordered(), ModelSQL, ModelView, ChatMixin):
cursor = Transaction().connection.cursor()
costs = defaultdict(Decimal)
- for sub_works in grouped_slice(works):
- red_sql = fields.SQL_OPERATORS['in'](
- cycle.work, [w.id for w in sub_works])
- query = cycle.select(
- cycle.work, Sum(Coalesce(cycle.cost, 0)).as_('cost'),
- where=red_sql & (cycle.state == 'done'),
- group_by=cycle.work)
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- costs.update(cursor)
- for cost in costs:
- costs[cost] = round_price(costs[cost])
+ query = cycle.select(
+ cycle.work, Sum(Coalesce(cycle.cost, 0)).as_('cost'),
+ where=fields.SQL_OPERATORS['in'](cycle.work, map(int, works))
+ & (cycle.state == 'done'),
+ group_by=cycle.work)
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ for work_id, cost in cursor:
+ costs[work_id] = round_price(cost)
return costs
@classmethod
diff --git a/modules/project_invoice/invoice.py b/modules/project_invoice/invoice.py
index 3a5bd5ae86..1f5f98e92e 100644
--- a/modules/project_invoice/invoice.py
+++ b/modules/project_invoice/invoice.py
@@ -10,7 +10,6 @@ from trytond.model import fields
from trytond.modules.account_invoice.exceptions import (
InvoiceLineValidationError)
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -90,18 +89,16 @@ class InvoiceLine(metaclass=PoolMeta):
durations = dict.fromkeys(map(int, lines))
query = ts_line.select(
ts_line.invoice_line, Sum(ts_line.duration),
+ where=fields.SQL_OPERATORS['in'](
+ ts_line.invoice_line, map(int, lines)),
group_by=ts_line.invoice_line)
- for sub_lines in grouped_slice(lines):
- query.where = fields.SQL_OPERATORS['in'](
- ts_line.invoice_line, map(int, sub_lines))
- cursor.execute(*query)
-
- for line_id, duration in cursor:
- # SQLite uses float for SUM
- if (duration is not None
- and not isinstance(duration, dt.timedelta)):
- duration = dt.timedelta(seconds=duration)
- durations[line_id] = duration
+ cursor.execute(*query)
+ for line_id, duration in cursor:
+ # SQLite uses float for SUM
+ if (duration is not None
+ and not isinstance(duration, dt.timedelta)):
+ duration = dt.timedelta(seconds=duration)
+ durations[line_id] = duration
return durations
@classmethod
diff --git a/modules/project_invoice/project.py b/modules/project_invoice/project.py
index 4491616a33..9dc9824080 100644
--- a/modules/project_invoice/project.py
+++ b/modules/project_invoice/project.py
@@ -18,7 +18,7 @@ from trytond.model.exceptions import AccessError
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, Id, If
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
from trytond.wizard import StateAction, Wizard
@@ -112,28 +112,26 @@ class Progress:
invoiced_progress = {}
quantities = {}
- for sub_works in grouped_slice(works):
- sub_works = list(sub_works)
- where = fields.SQL_OPERATORS['in'](
- table.id, [x.id for x in sub_works if x.invoice_unit_price])
- cursor.execute(*table.join(progress,
- condition=progress.work == table.id
- ).select(table.id, Sum(progress.progress),
- where=where,
- group_by=table.id))
- invoiced_progress.update(dict(cursor))
+ where = fields.SQL_OPERATORS['in'](
+ table.id, [x.id for x in works if x.invoice_unit_price])
+ cursor.execute(*table.join(progress,
+ condition=progress.work == table.id
+ ).select(table.id, Sum(progress.progress),
+ where=where,
+ group_by=table.id))
+ invoiced_progress.update(dict(cursor))
- for work in sub_works:
- delta = (
- (work.progress or 0)
- - invoiced_progress.get(work.id, 0.0))
- if work.invoice_unit_price and delta > 0:
- quantity = delta
- if work.price_list_hour:
- quantity *= work.effort_hours
- if work.unit_to_invoice:
- quantity = work.unit_to_invoice.round(quantity)
- quantities[work.id] = quantity
+ for work in works:
+ delta = (
+ (work.progress or 0)
+ - invoiced_progress.get(work.id, 0.0))
+ if work.invoice_unit_price and delta > 0:
+ quantity = delta
+ if work.price_list_hour:
+ quantity *= work.effort_hours
+ if work.unit_to_invoice:
+ quantity = work.unit_to_invoice.round(quantity)
+ quantities[work.id] = quantity
return quantities
@property
@@ -161,32 +159,31 @@ class Progress:
amounts = defaultdict(Decimal)
work2currency = {}
ids2work = dict((w.id, w) for w in works)
- for sub_ids in grouped_slice(ids2work.keys()):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- query = (table.join(progress,
- condition=progress.work == table.id
- ).join(invoice_line,
- condition=progress.invoice_line == invoice_line.id
- ).select(
- table.id,
- Sum(Cast(progress.progress, 'NUMERIC')
- * invoice_line.unit_price).as_('amount'),
- where=where,
- group_by=table.id))
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- for work_id, amount in cursor:
- work = ids2work[work_id]
- if work.price_list_hour:
- amount *= Decimal(str(work.effort_hours))
- amounts[work_id] = amount
+ where = fields.SQL_OPERATORS['in'](table.id, list(ids2work.keys()))
+ query = (table.join(progress,
+ condition=progress.work == table.id
+ ).join(invoice_line,
+ condition=progress.invoice_line == invoice_line.id
+ ).select(
+ table.id,
+ Sum(Cast(progress.progress, 'NUMERIC')
+ * invoice_line.unit_price).as_('amount'),
+ where=where,
+ group_by=table.id))
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ for work_id, amount in cursor:
+ work = ids2work[work_id]
+ if work.price_list_hour:
+ amount *= Decimal(str(work.effort_hours))
+ amounts[work_id] = amount
- cursor.execute(*table.join(company,
- condition=table.company == company.id
- ).select(table.id, company.currency,
- where=where))
- work2currency.update(cursor)
+ cursor.execute(*table.join(company,
+ condition=table.company == company.id
+ ).select(table.id, company.currency,
+ where=where))
+ work2currency.update(cursor)
currencies = Currency.browse(set(work2currency.values()))
id2currency = {c.id: c for c in currencies}
@@ -274,20 +271,19 @@ class Timesheet:
line.work, Sum(line.duration),
group_by=line.work)
for upto, tworks in upto2tworks.items():
- for sub_ids in grouped_slice(tworks):
- query.where = (
- fields.SQL_OPERATORS['in'](line.work, sub_ids)
- & (line.invoice_line == Null))
- if upto:
- query.where &= (line.date <= upto)
- cursor.execute(*query)
+ query.where = (
+ fields.SQL_OPERATORS['in'](line.work, tworks)
+ & (line.invoice_line == Null))
+ if upto:
+ query.where &= (line.date <= upto)
+ cursor.execute(*query)
- for twork_id, duration in cursor:
- if duration:
- # SQLite uses float for SUM
- if not isinstance(duration, datetime.timedelta):
- duration = datetime.timedelta(seconds=duration)
- durations[twork2work[twork_id]] += duration
+ for twork_id, duration in cursor:
+ if duration:
+ # SQLite uses float for SUM
+ if not isinstance(duration, datetime.timedelta):
+ duration = datetime.timedelta(seconds=duration)
+ durations[twork2work[twork_id]] += duration
quantities = {}
for work in works:
@@ -317,28 +313,26 @@ class Timesheet:
amounts = {}
work2currency = {}
- work_ids = [w.id for w in works]
- for sub_ids in grouped_slice(work_ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*table.join(timesheet_work,
- condition=(
- Concat(cls.__name__ + ',', table.id)
- == timesheet_work.origin)
- ).join(timesheet_line,
- condition=timesheet_line.work == timesheet_work.id
- ).join(invoice_line,
- condition=timesheet_line.invoice_line == invoice_line.id
- ).select(table.id,
- Sum(timesheet_line.duration * invoice_line.unit_price),
- where=where,
- group_by=table.id))
- amounts.update(cursor)
+ where = fields.SQL_OPERATORS['in'](table.id, map(int, works))
+ cursor.execute(*table.join(timesheet_work,
+ condition=(
+ Concat(cls.__name__ + ',', table.id)
+ == timesheet_work.origin)
+ ).join(timesheet_line,
+ condition=timesheet_line.work == timesheet_work.id
+ ).join(invoice_line,
+ condition=timesheet_line.invoice_line == invoice_line.id
+ ).select(table.id,
+ Sum(timesheet_line.duration * invoice_line.unit_price),
+ where=where,
+ group_by=table.id))
+ amounts.update(cursor)
- cursor.execute(*table.join(company,
- condition=table.company == company.id
- ).select(table.id, company.currency,
- where=where))
- work2currency.update(cursor)
+ cursor.execute(*table.join(company,
+ condition=table.company == company.id
+ ).select(table.id, company.currency,
+ where=where))
+ work2currency.update(cursor)
currencies = Currency.browse(set(work2currency.values()))
id2currency = {c.id: c for c in currencies}
diff --git a/modules/project_revenue/work.py b/modules/project_revenue/work.py
index 28cb08f4ef..55ca6a415e 100644
--- a/modules/project_revenue/work.py
+++ b/modules/project_revenue/work.py
@@ -13,7 +13,6 @@ from trytond.modules.currency.fields import Monetary
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -93,23 +92,21 @@ class Work(metaclass=PoolMeta):
work = Work.__table__()
line = Line.__table__()
- work_ids = [w.id for w in works]
- for sub_ids in grouped_slice(work_ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*table.join(work,
- condition=(
- Concat(cls.__name__ + ',', table.id) == work.origin)
- ).join(line, condition=line.work == work.id
- ).select(table.id, Sum(line.cost_price * line.duration),
- where=where,
- group_by=[table.id]))
- for work_id, cost in cursor:
- # SQLite stores timedelta as float
- if isinstance(cost, dt.timedelta):
- cost = cost.total_seconds()
- # Convert from seconds
- cost /= 60 * 60
- yield work_id, Decimal(str(cost))
+ where = fields.SQL_OPERATORS['in'](table.id, map(int, works))
+ cursor.execute(*table.join(work,
+ condition=(
+ Concat(cls.__name__ + ',', table.id) == work.origin)
+ ).join(line, condition=line.work == work.id
+ ).select(table.id, Sum(line.cost_price * line.duration),
+ where=where,
+ group_by=[table.id]))
+ for work_id, cost in cursor:
+ # SQLite stores timedelta as float
+ if isinstance(cost, dt.timedelta):
+ cost = cost.total_seconds()
+ # Convert from seconds
+ cost /= 60 * 60
+ yield work_id, Decimal(str(cost))
@classmethod
def _purchase_cost(cls, works):
@@ -131,27 +128,25 @@ class Work(metaclass=PoolMeta):
invoice = Invoice.__table__()
company = Company.__table__()
- work_ids = [w.id for w in works]
work2currency = {}
iline2work = {}
- for sub_ids in grouped_slice(work_ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*table.join(purchase_line,
- condition=purchase_line.work == table.id
- ).join(invoice_line,
- condition=invoice_line.origin == Concat(
- 'purchase.line,', purchase_line.id)
- ).join(invoice,
- condition=invoice_line.invoice == invoice.id
- ).select(invoice_line.id, table.id,
- where=where & ~invoice.state.in_(['draft', 'cancelled'])))
- iline2work.update(cursor)
+ where = fields.SQL_OPERATORS['in'](table.id, map(int, works))
+ cursor.execute(*table.join(purchase_line,
+ condition=purchase_line.work == table.id
+ ).join(invoice_line,
+ condition=invoice_line.origin == Concat(
+ 'purchase.line,', purchase_line.id)
+ ).join(invoice,
+ condition=invoice_line.invoice == invoice.id
+ ).select(invoice_line.id, table.id,
+ where=where & ~invoice.state.in_(['draft', 'cancelled'])))
+ iline2work.update(cursor)
- cursor.execute(*table.join(company,
- condition=table.company == company.id
- ).select(table.id, company.currency,
- where=where))
- work2currency.update(cursor)
+ cursor.execute(*table.join(company,
+ condition=table.company == company.id
+ ).select(table.id, company.currency,
+ where=where))
+ work2currency.update(cursor)
currencies = Currency.browse(set(work2currency.values()))
id2currency = {c.id: c for c in currencies}
diff --git a/modules/purchase/product.py b/modules/purchase/product.py
index 07d240717c..fdae36a308 100644
--- a/modules/purchase/product.py
+++ b/modules/purchase/product.py
@@ -13,7 +13,7 @@ from trytond.modules.product import (
ProductDeactivatableMixin, price_digits, round_price)
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, If, TimeDelta
-from trytond.tools import grouped_slice, is_full_text, lstrip_wildcard
+from trytond.tools import is_full_text, lstrip_wildcard
from trytond.transaction import Transaction
from .exceptions import PurchaseUOMWarning
@@ -154,16 +154,15 @@ class Product(metaclass=PoolMeta):
supplier = context.get('supplier')
if supplier:
where &= purchase.party == supplier
- for sub_products in grouped_slice(products):
- query = (line
- .join(purchase, condition=line.purchase == purchase.id)
- .select(
- Max(line.id),
- where=where & fields.SQL_OPERATORS['in'](
- line.product, map(int, sub_products)),
- group_by=[line.product]))
- cursor.execute(*query)
- line_ids.extend(i for i, in cursor)
+ query = (line
+ .join(purchase, condition=line.purchase == purchase.id)
+ .select(
+ Max(line.id),
+ where=where & fields.SQL_OPERATORS['in'](
+ line.product, map(int, products)),
+ group_by=[line.product]))
+ cursor.execute(*query)
+ line_ids.extend(i for i, in cursor)
lines = Line.browse(line_ids)
uom = None
diff --git a/modules/purchase_history/purchase.py b/modules/purchase_history/purchase.py
index f4f0dbc7f7..4434058c77 100644
--- a/modules/purchase_history/purchase.py
+++ b/modules/purchase_history/purchase.py
@@ -4,7 +4,6 @@
from trytond.model import ModelView, Workflow, fields
from trytond.pool import PoolMeta
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -45,12 +44,11 @@ class Purchase(metaclass=PoolMeta):
table = cls.__table__()
# Use SQL and before super to avoid two history entries
- for sub_purchases in grouped_slice(purchases):
- cursor.execute(*table.update(
- [table.revision],
- [table.revision + 1],
- where=fields.SQL_OPERATORS['in'](
- table.id, map(int, sub_purchases))))
+ cursor.execute(*table.update(
+ [table.revision],
+ [table.revision + 1],
+ where=fields.SQL_OPERATORS['in'](
+ table.id, map(int, purchases))))
super().draft(purchases)
diff --git a/modules/sale/sale_reporting.py b/modules/sale/sale_reporting.py
index ace2085f00..ebe2daae35 100644
--- a/modules/sale/sale_reporting.py
+++ b/modules/sale/sale_reporting.py
@@ -15,7 +15,7 @@ from trytond.model import ModelSQL, ModelView, UnionMixin, fields, sum_tree
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Eval, If
-from trytond.tools import grouped_slice, pairwise_longest
+from trytond.tools import pairwise_longest
from trytond.tools.chart import sparkline
from trytond.transaction import Transaction
from trytond.wizard import StateAction, StateTransition, Wizard
@@ -513,16 +513,13 @@ class CustomerCategoryTree(ModelSQL, ModelView):
categories = cls.search([
('parent', 'child_of', [c.id for c in categories]),
])
- ids = [c.id for c in categories]
reporting_customer_categories = []
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- where = fields.SQL_OPERATORS['in'](
- reporting_product_category.id, sub_ids)
- cursor.execute(
- *reporting_product_category.select(
- reporting_product_category.id, where=where))
- reporting_customer_categories.extend(r for r, in cursor)
+ where = fields.SQL_OPERATORS['in'](
+ reporting_product_category.id, map(int, categories))
+ cursor.execute(
+ *reporting_product_category.select(
+ reporting_product_category.id, where=where))
+ reporting_customer_categories.extend(r for r, in cursor)
result = {}
reporting_product_categories = ReportingCustomerCategory.browse(
@@ -755,16 +752,13 @@ class ProductCategoryTree(ModelSQL, ModelView):
categories = cls.search([
('parent', 'child_of', [c.id for c in categories]),
])
- ids = [c.id for c in categories]
reporting_product_categories = []
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- where = fields.SQL_OPERATORS['in'](
- reporting_product_category.id, sub_ids)
- cursor.execute(
- *reporting_product_category.select(
- reporting_product_category.id, where=where))
- reporting_product_categories.extend(r for r, in cursor)
+ where = fields.SQL_OPERATORS['in'](
+ reporting_product_category.id, map(int, categories))
+ cursor.execute(
+ *reporting_product_category.select(
+ reporting_product_category.id, where=where))
+ reporting_product_categories.extend(r for r, in cursor)
result = {}
reporting_product_categories = ReportingProductCategory.browse(
diff --git a/modules/sale_complaint/complaint.py b/modules/sale_complaint/complaint.py
index 21094680cb..ac7c301536 100644
--- a/modules/sale_complaint/complaint.py
+++ b/modules/sale_complaint/complaint.py
@@ -8,6 +8,7 @@ from decimal import Decimal
from sql import Null
from sql.functions import CharLength
+from trytond import backend
from trytond.i18n import gettext
from trytond.model import (
ChatMixin, DeactivableMixin, Index, ModelSQL, ModelView, Workflow, fields)
@@ -392,7 +393,8 @@ class Complaint(Workflow, ModelSQL, ModelView, ChatMixin):
def _check_similar(cls, complaints):
pool = Pool()
Warning = pool.get('res.user.warning')
- for sub_complaints in grouped_slice(complaints):
+ for sub_complaints in grouped_slice(
+ complaints, backend.MAX_QUERY_PARAMS // 10):
sub_complaints = list(sub_complaints)
domain = list(filter(None,
(c._similar_domain() for c in sub_complaints)))
diff --git a/modules/sale_credit_limit/party.py b/modules/sale_credit_limit/party.py
index 589966252f..b0d3470414 100644
--- a/modules/sale_credit_limit/party.py
+++ b/modules/sale_credit_limit/party.py
@@ -4,7 +4,6 @@ from decimal import Decimal
from itertools import groupby
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -22,43 +21,42 @@ class Party(metaclass=PoolMeta):
company_id = Transaction().context.get('company')
- for sub_parties in grouped_slice(parties):
- id2party = {p.id: p for p in sub_parties}
+ id2party = {p.id: p for p in parties}
- sales = Sale.search([
- ('company', '=', company_id),
- ('party', 'in', list(id2party.keys())),
- ('state', 'in', ['confirmed', 'processing']),
- ],
- order=[('party', None)])
- for party_id, sales in groupby(sales, lambda s: s.party.id):
- party = id2party[party_id]
- for sale in sales:
- amount = 0
- for line in sale.lines:
- quantity = line.credit_limit_quantity
- if quantity is None:
+ sales = Sale.search([
+ ('company', '=', company_id),
+ ('party', 'in', list(id2party.keys())),
+ ('state', 'in', ['confirmed', 'processing']),
+ ],
+ order=[('party', None)])
+ for party_id, sales in groupby(sales, lambda s: s.party.id):
+ party = id2party[party_id]
+ for sale in sales:
+ amount = 0
+ for line in sale.lines:
+ quantity = line.credit_limit_quantity
+ if quantity is None:
+ continue
+ for invoice_line in line.invoice_lines:
+ if invoice_line.type != 'line':
continue
- for invoice_line in line.invoice_lines:
- if invoice_line.type != 'line':
- continue
- invoice = invoice_line.invoice
- if invoice and invoice.move:
- if line.unit:
- quantity -= Uom.compute_qty(
- invoice_line.unit or line.unit,
- invoice_line.quantity,
- line.unit, round=False)
- else:
- quantity -= invoice_line.quantity
- if quantity > 0:
- amount += Currency.compute(
- sale.currency,
- Decimal(str(quantity)) * line.unit_price,
- party.currency,
- round=False)
- amounts[party.id] = party.currency.round(
- amounts[sale.party.id] + amount)
+ invoice = invoice_line.invoice
+ if invoice and invoice.move:
+ if line.unit:
+ quantity -= Uom.compute_qty(
+ invoice_line.unit or line.unit,
+ invoice_line.quantity,
+ line.unit, round=False)
+ else:
+ quantity -= invoice_line.quantity
+ if quantity > 0:
+ amount += Currency.compute(
+ sale.currency,
+ Decimal(str(quantity)) * line.unit_price,
+ party.currency,
+ round=False)
+ amounts[party.id] = party.currency.round(
+ amounts[sale.party.id] + amount)
return amounts
@classmethod
diff --git a/modules/sale_history/sale.py b/modules/sale_history/sale.py
index dd7864757f..0dd7798ce0 100644
--- a/modules/sale_history/sale.py
+++ b/modules/sale_history/sale.py
@@ -4,7 +4,6 @@
from trytond.model import ModelView, Workflow, fields
from trytond.pool import PoolMeta
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -41,12 +40,11 @@ class RevisionMixin:
table = cls.__table__()
# Use SQL and before super to avoid two history entries
- for sub_records in grouped_slice(records):
- cursor.execute(*table.update(
- [table.revision],
- [table.revision + 1],
- where=fields.SQL_OPERATORS['in'](
- table.id, map(int, sub_records))))
+ cursor.execute(*table.update(
+ [table.revision],
+ [table.revision + 1],
+ where=fields.SQL_OPERATORS['in'](
+ table.id, map(int, records))))
super().draft(records)
diff --git a/modules/sale_point/sale.py b/modules/sale_point/sale.py
index 2c626014a3..c4d13a37b6 100644
--- a/modules/sale_point/sale.py
+++ b/modules/sale_point/sale.py
@@ -21,7 +21,7 @@ from trytond.modules.currency.fields import Monetary
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, Id, If
-from trytond.tools import grouped_slice, sqlite_apply_types
+from trytond.tools import sqlite_apply_types
from trytond.transaction import Transaction
from trytond.wizard import Button, StateTransition, StateView, Wizard
@@ -82,15 +82,14 @@ class POS(ModelSQL, ModelView):
if mode == 'write':
if 'tax_included' in values:
- for sub_points in grouped_slice(points):
- if Sale.search([
- ('point', 'in',
- list(map(int, sub_points))),
- ],
- limit=1, order=[]):
- raise AccessError(gettext(
- 'sale_point.'
- 'msg_point_change_tax_included'))
+ if Sale.search([
+ ('point', 'in',
+ list(map(int, points))),
+ ],
+ limit=1, order=[]):
+ raise AccessError(gettext(
+ 'sale_point.'
+ 'msg_point_change_tax_included'))
class POSSale(Workflow, ModelSQL, ModelView, TaxableMixin, ChatMixin):
@@ -859,29 +858,28 @@ class POSCashSession(Workflow, ModelSQL, ModelView):
transfer = Transfer.__table__()
cursor = Transaction().connection.cursor()
balances = defaultdict(Decimal)
- for sub_sessions in grouped_slice(sessions):
- sub_ids = [s.id for s in sub_sessions]
- query = Union(
- payment.select(
- payment.session.as_('session'),
- Sum(payment.amount).as_('amount'),
- where=fields.SQL_OPERATORS['in'](
- payment.session, sub_ids),
- group_by=[payment.session]),
- transfer.select(
- transfer.session.as_('session'),
- Sum(transfer.amount).as_('amount'),
- where=fields.SQL_OPERATORS['in'](
- transfer.session, sub_ids),
- group_by=[transfer.session]),
- all_=True)
- query = query.select(
- query.session, Sum(query.amount).as_('amount'),
- group_by=[query.session])
- if backend.name == 'sqlite':
- sqlite_apply_types(query, [None, 'NUMERIC'])
- cursor.execute(*query)
- balances.update(cursor)
+ session_ids = [s.id for s in sessions]
+ query = Union(
+ payment.select(
+ payment.session.as_('session'),
+ Sum(payment.amount).as_('amount'),
+ where=fields.SQL_OPERATORS['in'](
+ payment.session, session_ids),
+ group_by=[payment.session]),
+ transfer.select(
+ transfer.session.as_('session'),
+ Sum(transfer.amount).as_('amount'),
+ where=fields.SQL_OPERATORS['in'](
+ transfer.session, session_ids),
+ group_by=[transfer.session]),
+ all_=True)
+ query = query.select(
+ query.session, Sum(query.amount).as_('amount'),
+ group_by=[query.session])
+ if backend.name == 'sqlite':
+ sqlite_apply_types(query, [None, 'NUMERIC'])
+ cursor.execute(*query)
+ balances.update(cursor)
for session in sessions:
balances[session.id] = session.currency.round(balances[session.id])
return balances
diff --git a/modules/sale_promotion_coupon/sale.py b/modules/sale_promotion_coupon/sale.py
index 48cd4ff08c..3392dc43e0 100644
--- a/modules/sale_promotion_coupon/sale.py
+++ b/modules/sale_promotion_coupon/sale.py
@@ -15,7 +15,6 @@ from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval, If
from trytond.sql.functions import DateRange
from trytond.sql.operators import RangeOverlap
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .exceptions import PromotionCouponNumberDatesError
@@ -299,14 +298,10 @@ class PromotionCouponNumber(DeactivableMixin, ModelSQL, ModelView):
query, table, active = cls._active_query()
query.columns = [table.id, active]
-
- result = {}
- for sub_numbers in grouped_slice(numbers):
- query.where = fields.SQL_OPERATORS['in'](
- table.id, map(int, sub_numbers))
- cursor.execute(*query)
- result.update(dict(cursor))
- return result
+ query.where = fields.SQL_OPERATORS['in'](
+ table.id, map(int, numbers))
+ cursor.execute(*query)
+ return dict(cursor)
@classmethod
def search_active(cls, name, clause):
diff --git a/modules/sale_stock_quantity/sale.py b/modules/sale_stock_quantity/sale.py
index 0e8738c994..125a0771cd 100644
--- a/modules/sale_stock_quantity/sale.py
+++ b/modules/sale_stock_quantity/sale.py
@@ -9,7 +9,7 @@ from operator import attrgetter
from trytond.i18n import gettext
from trytond.model import ModelView, Workflow, fields
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice, sortable_values
+from trytond.tools import sortable_values
from trytond.transaction import Transaction
from .exceptions import StockQuantityError, StockQuantityWarning
@@ -104,12 +104,11 @@ class Sale(metaclass=PoolMeta):
stock_date_start=date,
stock_date_end=date):
pbl = defaultdict(int)
- for sub_products in grouped_slice(products):
- sub_product_ids = [p.id for p in sub_products]
- pbl.update(Product.products_by_location(
- [warehouse.id],
- with_childs=True,
- grouping_filter=(sub_product_ids,)))
+ product_ids = [p.id for p in products]
+ pbl.update(Product.products_by_location(
+ [warehouse.id],
+ with_childs=True,
+ grouping_filter=(product_ids,)))
delta = {}
for key, qty in pbl.items():
_, product_id = key
@@ -153,27 +152,26 @@ class Sale(metaclass=PoolMeta):
(p, p.forecast_quantity) for p in products)
# Remove quantities from other sales
- for sub_products in grouped_slice(products):
- other_lines = Line.search([
- ('sale.company', '=', self.company.id),
- ('sale.state', 'in', self._stock_quantity_states()),
- ('sale.id', '!=', self.id),
- ('product', 'in', sub_products),
- ('quantity', '>', 0),
- ])
- for line in other_lines:
- if line.warehouse != warehouse:
- continue
- if (line.shipping_date
- and line.shipping_date > shipping_date):
- continue
- product = line.product
- date = line.sale.sale_date or today
- if date > today:
- continue
- quantity = Uom.compute_qty(line.unit, line.quantity,
- product.default_uom, round=False)
- quantities[line.warehouse][product] -= quantity
+ other_lines = Line.search([
+ ('sale.company', '=', self.company.id),
+ ('sale.state', 'in', self._stock_quantity_states()),
+ ('sale.id', '!=', self.id),
+ ('product', 'in', products),
+ ('quantity', '>', 0),
+ ])
+ for line in other_lines:
+ if line.warehouse != warehouse:
+ continue
+ if (line.shipping_date
+ and line.shipping_date > shipping_date):
+ continue
+ product = line.product
+ date = line.sale.sale_date or today
+ if date > today:
+ continue
+ quantity = Uom.compute_qty(line.unit, line.quantity,
+ product.default_uom, round=False)
+ quantities[line.warehouse][product] -= quantity
for line in lines:
warehouse = line.warehouse
diff --git a/modules/sale_subscription_asset/stock.py b/modules/sale_subscription_asset/stock.py
index b9405f0c13..957cd4d949 100644
--- a/modules/sale_subscription_asset/stock.py
+++ b/modules/sale_subscription_asset/stock.py
@@ -1,8 +1,10 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
+
+from collections import defaultdict
+
from trytond.model import fields
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -24,20 +26,19 @@ class Lot(metaclass=PoolMeta):
Date = pool.get('ir.date')
SubscriptionLine = pool.get('sale.subscription.line')
- subscribed_lines = {l.id: None for l in lots}
+ subscribed_lines = defaultdict(lambda: None)
date = Transaction().context.get('date', Date.today())
- for sub_lots in grouped_slice(lots):
- lines = SubscriptionLine.search([
- ('asset_lot', 'in', [l.id for l in sub_lots]),
- [
- ('start_date', '<=', date),
- ['OR',
- ('end_date', '=', None),
- ('end_date', '>', date),
- ],
- ]
- ])
- subscribed_lines.update((s.asset_lot.id, s.id) for s in lines)
+ lines = SubscriptionLine.search([
+ ('asset_lot', 'in', lots),
+ [
+ ('start_date', '<=', date),
+ ['OR',
+ ('end_date', '=', None),
+ ('end_date', '>', date),
+ ],
+ ]
+ ])
+ subscribed_lines.update((s.asset_lot.id, s.id) for s in lines)
return subscribed_lines
@classmethod
diff --git a/modules/sale_supply_drop_shipment/product.py b/modules/sale_supply_drop_shipment/product.py
index fbd774b8fa..9354c4e5e7 100644
--- a/modules/sale_supply_drop_shipment/product.py
+++ b/modules/sale_supply_drop_shipment/product.py
@@ -1,7 +1,6 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
class Product(metaclass=PoolMeta):
@@ -13,17 +12,16 @@ class Product(metaclass=PoolMeta):
Move = pool.get('stock.move')
Shipment = pool.get('stock.shipment.drop')
shipments = set()
- for sub_products in grouped_slice(products):
- domain = [
- ('unit_price_updated', '=', True),
- cls._domain_moves_cost(),
- ('product', 'in', [p.id for p in sub_products]),
- ('shipment', 'like', 'stock.shipment.drop,%'),
- ]
- if start:
- domain.append(('effective_date', '>=', start))
- for move in Move.search(domain, order=[]):
- shipments.add(move.shipment)
+ domain = [
+ ('unit_price_updated', '=', True),
+ cls._domain_moves_cost(),
+ ('product', 'in', products),
+ ('shipment', 'like', 'stock.shipment.drop,%'),
+ ]
+ if start:
+ domain.append(('effective_date', '>=', start))
+ for move in Move.search(domain, order=[]):
+ shipments.add(move.shipment)
shipments = Shipment.browse(list(shipments))
Shipment.set_cost(shipments)
super().recompute_cost_price(products, start=start)
diff --git a/modules/sale_supply_production/production.py b/modules/sale_supply_production/production.py
index 8966526207..bc47e40cab 100644
--- a/modules/sale_supply_production/production.py
+++ b/modules/sale_supply_production/production.py
@@ -7,7 +7,6 @@ from trytond.i18n import gettext
from trytond.model import Model, ModelView, Workflow
from trytond.model.exceptions import AccessError
from trytond.pool import Pool, PoolMeta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction, without_check_access
@@ -21,11 +20,9 @@ def process_sale_supply(func):
sales = set()
with without_check_access():
- for sub_productions in grouped_slice(productions):
- ids = [p.id for p in sub_productions]
- sales.update([s.id for s in Sale.search([
- ('lines.productions', 'in', ids),
- ])])
+ sales.update([s.id for s in Sale.search([
+ ('lines.productions', 'in', productions),
+ ])])
result = func(cls, productions)
if sales:
with transaction.set_context(
diff --git a/modules/stock/inventory.py b/modules/stock/inventory.py
index 4051a72b2d..ee2a450fec 100644
--- a/modules/stock/inventory.py
+++ b/modules/stock/inventory.py
@@ -11,7 +11,7 @@ from trytond.model import (
from trytond.model.exceptions import AccessError
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If
-from trytond.tools import grouped_slice, is_full_text, lstrip_wildcard
+from trytond.tools import is_full_text, lstrip_wildcard
from trytond.transaction import Transaction
from trytond.wizard import Button, StateTransition, StateView, Wizard
@@ -257,11 +257,10 @@ class Inventory(Workflow, ModelSQL, ModelView, ChatMixin):
else:
product_ids = [l.product.id for l in inventory.lines]
pbl = defaultdict(int)
- for product_ids in grouped_slice(product_ids):
- pbl.update(Product.products_by_location(
- [inventory.location.id],
- grouping=grouping,
- grouping_filter=(list(product_ids),)))
+ pbl.update(Product.products_by_location(
+ [inventory.location.id],
+ grouping=grouping,
+ grouping_filter=(list(product_ids),)))
# Update existing lines
for line in inventory.lines:
diff --git a/modules/stock/location.py b/modules/stock/location.py
index 99cf8a9638..939f1dc3bd 100644
--- a/modules/stock/location.py
+++ b/modules/stock/location.py
@@ -14,7 +14,6 @@ from trytond.model import (
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool
from trytond.pyson import Eval, If, TimeDelta
-from trytond.tools import grouped_slice
from trytond.transaction import (
Transaction, inactive_records, without_check_access)
@@ -314,18 +313,16 @@ class Location(DeactivableMixin, tree(), ModelSQL, ModelView):
for (location_id, product), quantity in quantities.items():
if quantity and product not in consumables:
empty.discard(location_id)
- for sub_ids in grouped_slice(list(empty)):
- sub_ids = list(sub_ids)
- moves = Move.search([
- ('state', 'not in', ['done', 'cancelled']),
- ['OR',
- ('from_location', 'in', sub_ids),
- ('to_location', 'in', sub_ids),
- ],
- ])
- for move in moves:
- for location in [move.from_location, move.to_location]:
- empty.discard(location.id)
+ moves = Move.search([
+ ('state', 'not in', ['done', 'cancelled']),
+ ['OR',
+ ('from_location', 'in', empty),
+ ('to_location', 'in', empty),
+ ],
+ ])
+ for move in moves:
+ for location in [move.from_location, move.to_location]:
+ empty.discard(location.id)
return cls.browse(empty)
@staticmethod
@@ -434,15 +431,13 @@ class Location(DeactivableMixin, tree(), ModelSQL, ModelView):
if not grouping:
return {loc.id: None for loc in locations}
- pbl = {}
- for sub_locations in grouped_slice(locations):
- location_ids = [l.id for l in sub_locations]
- with Transaction().set_context(context):
- pbl.update(Product.products_by_location(
- location_ids,
- grouping=grouping,
- grouping_filter=grouping_filter,
- with_childs=trans_context.get('with_childs', True)))
+ location_ids = [l.id for l in locations]
+ with Transaction().set_context(context):
+ pbl = Product.products_by_location(
+ location_ids,
+ grouping=grouping,
+ grouping_filter=grouping_filter,
+ with_childs=trans_context.get('with_childs', True))
return dict((loc.id, pbl.get((loc.id,) + key, 0)) for loc in locations)
diff --git a/modules/stock/message.xml b/modules/stock/message.xml
index 4c38169176..469144401d 100644
--- a/modules/stock/message.xml
+++ b/modules/stock/message.xml
@@ -13,9 +13,6 @@ this repository contains the full copyright notices and license terms. -->
You cannot change the cost price of a product which is associated with stock moves.
You must use the "Modify Cost Price" wizard.
-
- You cannot select more than %(max)s products to compute quantities.
-
Invalid cost price "%(cost_price)s" for product "%(product)s" with exception "%(exception)s".
diff --git a/modules/stock/move.py b/modules/stock/move.py
index 5781892d16..4e33df0141 100644
--- a/modules/stock/move.py
+++ b/modules/stock/move.py
@@ -19,7 +19,7 @@ from trytond.model.exceptions import AccessError
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, Id, If, TimeDelta
-from trytond.tools import cached_property, grouped_slice
+from trytond.tools import cached_property
from trytond.transaction import Transaction, without_check_access
from .exceptions import MoveFutureWarning, MoveOriginWarning
@@ -1314,7 +1314,6 @@ class Move(Workflow, ModelSQL, ModelView):
connection = transaction.connection
if database.has_select_for():
- count = database.IN_MAX // 2
PeriodCache = Period.get_cache(grouping)
with connection.cursor() as cursor:
for company_id in company_ids:
@@ -1327,24 +1326,22 @@ class Move(Workflow, ModelSQL, ModelView):
], order=[('date', 'DESC')], limit=1)
if periods:
period, = periods
- for sub_location_ids in grouped_slice(location_ids, count):
- sub_location_ids = list(sub_location_ids)
- table = cls.__table__()
- query = table.select(Literal(1),
- where=(fields.SQL_OPERATORS['in'](
- table.to_location, sub_location_ids)
- | fields.SQL_OPERATORS['in'](
- table.from_location, sub_location_ids))
- & table.product.in_(product_ids)
- & (table.company == company_id),
- for_=For('UPDATE', nowait=True))
+ table = cls.__table__()
+ query = table.select(Literal(1),
+ where=(fields.SQL_OPERATORS['in'](
+ table.to_location, location_ids)
+ | fields.SQL_OPERATORS['in'](
+ table.from_location, location_ids))
+ & table.product.in_(product_ids)
+ & (table.company == company_id),
+ for_=For('UPDATE', nowait=True))
- if period:
- query.where &= Coalesce(
- table.effective_date,
- table.planned_date,
- datetime.date.max) > period.date
- cursor.execute(*query)
+ if period:
+ query.where &= Coalesce(
+ table.effective_date,
+ table.planned_date,
+ datetime.date.max) > period.date
+ cursor.execute(*query)
else:
cls.lock()
diff --git a/modules/stock/period.py b/modules/stock/period.py
index 1ef8c6d3d0..8f6d2bbc06 100644
--- a/modules/stock/period.py
+++ b/modules/stock/period.py
@@ -1,6 +1,6 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-from itertools import chain, groupby
+from itertools import groupby
from sql import For, Literal
@@ -8,7 +8,6 @@ from trytond.i18n import gettext
from trytond.model import Index, ModelSQL, ModelView, Workflow, fields
from trytond.pool import Pool
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .exceptions import PeriodCloseError
@@ -89,13 +88,10 @@ class Period(Workflow, ModelSQL, ModelView):
def draft(cls, periods):
for grouping in cls.groupings():
Cache = cls.get_cache(grouping)
- caches = []
- for sub_periods in grouped_slice(periods):
- caches.append(Cache.search([
- ('period', 'in',
- [p.id for p in sub_periods]),
- ], order=[]))
- Cache.delete(list(chain(*caches)))
+ caches = Cache.search([
+ ('period', 'in', periods),
+ ], order=[])
+ Cache.delete(caches)
@classmethod
@ModelView.button
diff --git a/modules/stock/product.py b/modules/stock/product.py
index b83301f37c..65b3f7e565 100644
--- a/modules/stock/product.py
+++ b/modules/stock/product.py
@@ -17,7 +17,7 @@ from trytond.model.exceptions import AccessError
from trytond.modules.product import price_digits, round_price
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, If, PYSONEncoder
-from trytond.tools import decistmt, grouped_slice
+from trytond.tools import decistmt
from trytond.tools import timezone as tz
from trytond.transaction import Transaction, without_check_access
from trytond.wizard import (
@@ -35,14 +35,11 @@ def _find_moves(cls, records):
field = 'product.template'
else:
field = 'product'
- for sub_records in grouped_slice(records):
- moves = Move.search([
- (field, 'in', list(map(int, sub_records))),
- ],
- limit=1, order=[])
- if moves:
- return moves
- return False
+ moves = Move.search([
+ (field, 'in', list(map(int, records))),
+ ],
+ limit=1, order=[])
+ return bool(moves)
def _check_no_stock(cls, records):
@@ -91,10 +88,9 @@ def _check_no_stock(cls, records):
locations = Location.search([('type', '=', 'storage')])
location_ids = list(map(int, locations))
for company in Company.search([]):
- for sub_records in grouped_slice(records):
- record2locations = get_record_locations(
- company, location_ids, sub_records, grouping)
- raise_warning(cls, company, record2locations)
+ record2locations = get_record_locations(
+ company, location_ids, records, grouping)
+ raise_warning(cls, company, record2locations)
class Template(metaclass=PoolMeta):
@@ -279,12 +275,11 @@ class Product(StockMixin, object, metaclass=PoolMeta):
pool = Pool()
Move = pool.get('stock.move')
deactivated = super().deactivate_replaced(products=products)
- for sub_products in grouped_slice(deactivated):
- moves = Move.search([
- ('product', 'in', list(map(int, sub_products))),
- ('state', 'in', ['staging', 'draft']),
- ])
- Move.delete(moves)
+ moves = Move.search([
+ ('product', 'in', list(map(int, products))),
+ ('state', 'in', ['staging', 'draft']),
+ ])
+ Move.delete(moves)
return deactivated
@classmethod
@@ -367,16 +362,14 @@ class Product(StockMixin, object, metaclass=PoolMeta):
cost = round_price(cost)
costs[cost].append(product)
- updated = []
- for sub_products in grouped_slice(products):
- domain = [
- ('unit_price_updated', '=', True),
- cls._domain_moves_cost(),
- ('product', 'in', [p.id for p in sub_products]),
- ]
- if start:
- domain.append(('effective_date', '>=', start))
- updated += Move.search(domain, order=[])
+ domain = [
+ ('unit_price_updated', '=', True),
+ cls._domain_moves_cost(),
+ ('product', 'in', products),
+ ]
+ if start:
+ domain.append(('effective_date', '>=', start))
+ updated = Move.search(domain, order=[])
if updated:
Move.write(updated, {'unit_price_updated': False})
@@ -675,10 +668,6 @@ class ProductQuantitiesByWarehouse(ModelSQL, ModelView):
product = Product.__table__()
from_ = move.join(product, condition=move.product == product.id)
if product_template:
- if len(product_template) > transaction.database.IN_MAX:
- raise AccessError(gettext(
- 'stock.msg_product_quantities_max',
- max=transaction.database.IN_MAX))
product_clause = fields.SQL_OPERATORS['in'](
product.template, product_template)
else:
@@ -692,10 +681,6 @@ class ProductQuantitiesByWarehouse(ModelSQL, ModelView):
if isinstance(product, int):
product = [product]
if product:
- if len(product) > transaction.database.IN_MAX:
- raise AccessError(gettext(
- 'stock.msg_product_quantities_max',
- max=transaction.database.IN_MAX))
product_clause = fields.SQL_OPERATORS['in'](
move.product, product)
else:
@@ -943,10 +928,6 @@ class ProductQuantitiesByWarehouseMove(ModelSQL, ModelView):
product = Product.__table__()
from_ = move.join(product, condition=move.product == product.id)
if product_template:
- if len(product_template) > transaction.database.IN_MAX:
- raise AccessError(gettext(
- 'stock.msg_product_quantities_max',
- max=transaction.database.IN_MAX))
product_clause = fields.SQL_OPERATORS['in'](
product.template, product_template)
else:
@@ -959,10 +940,6 @@ class ProductQuantitiesByWarehouseMove(ModelSQL, ModelView):
if isinstance(product, int):
product = [product]
if product:
- if len(product) > transaction.database.IN_MAX:
- raise AccessError(gettext(
- 'stock.msg_product_quantities_max',
- max=transaction.database.IN_MAX))
product_clause = fields.SQL_OPERATORS['in'](
move.product, product)
else:
@@ -1139,17 +1116,16 @@ class RecomputeCostPrice(Wizard):
products = []
from_ = None
- for sub_products in grouped_slice(products):
- moves = Move.search([
- ('unit_price_updated', '=', True),
- Product._domain_moves_cost(),
- ('product', 'in', [p.id for p in sub_products]),
- ],
- order=[('effective_date', 'ASC')],
- limit=1)
- if moves:
- move, = moves
- from_ = min(from_ or datetime.date.max, move.effective_date)
+ moves = Move.search([
+ ('unit_price_updated', '=', True),
+ Product._domain_moves_cost(),
+ ('product', 'in', products),
+ ],
+ order=[('effective_date', 'ASC')],
+ limit=1)
+ if moves:
+ move, = moves
+ from_ = min(from_ or datetime.date.max, move.effective_date)
return {'from_': from_}
def transition_recompute(self):
diff --git a/modules/stock/stock_reporting_margin.py b/modules/stock/stock_reporting_margin.py
index a587f406b6..60eb6fc564 100644
--- a/modules/stock/stock_reporting_margin.py
+++ b/modules/stock/stock_reporting_margin.py
@@ -15,7 +15,7 @@ from trytond.model import ModelSQL, ModelView, fields, sum_tree
from trytond.modules.currency.fields import Monetary
from trytond.pool import Pool
from trytond.pyson import Eval, If
-from trytond.tools import grouped_slice, pairwise_longest
+from trytond.tools import pairwise_longest
from trytond.tools.chart import sparkline
from trytond.transaction import Transaction
@@ -592,15 +592,11 @@ class CategoryTree(ModelSQL, ModelView):
categories = cls.search([
('parent', 'child_of', [c.id for c in categories]),
])
- ids = [c.id for c in categories]
- reporting_categories = []
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- where = fields.SQL_OPERATORS['in'](
- reporting_category.id, sub_ids)
- cursor.execute(
- *reporting_category.select(reporting_category.id, where=where))
- reporting_categories.extend(r for r, in cursor)
+ cursor.execute(*reporting_category.select(
+ reporting_category.id,
+ where=fields.SQL_OPERATORS['in'](
+ reporting_category.id, map(int, categories))))
+ reporting_categories = [r for r, in cursor]
result = {}
reporting_categories = ReportingCategory.browse(reporting_categories)
diff --git a/modules/stock_forecast/forecast.py b/modules/stock_forecast/forecast.py
index e5cb1d18c4..33bfeebf24 100644
--- a/modules/stock_forecast/forecast.py
+++ b/modules/stock_forecast/forecast.py
@@ -18,7 +18,6 @@ from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If
from trytond.sql.functions import DateRange
from trytond.sql.operators import RangeOverlap
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from trytond.wizard import Button, StateTransition, StateView, Wizard
@@ -355,31 +354,30 @@ class ForecastLine(ModelSQL, ModelView):
for forecast_id, lines in itertools.groupby(lines, key):
forecast = Forecast(forecast_id)
product2line = dict((line.product.id, line) for line in lines)
- product_ids = product2line.keys()
- for sub_ids in grouped_slice(product_ids):
- where = fields.SQL_OPERATORS['in'](move.product, sub_ids)
- cursor.execute(*move.join(location_from,
- condition=move.from_location == location_from.id
- ).join(location_to,
- condition=move.to_location == location_to.id
- ).select(move.product, Sum(move.internal_quantity),
- where=where
- & (location_from.left >= forecast.warehouse.left)
- & (location_from.right <= forecast.warehouse.right)
- & (location_to.left >= forecast.destination.left)
- & (location_to.right <= forecast.destination.right)
- & (move.state != 'cancelled')
- & (Coalesce(move.effective_date, move.planned_date)
- >= forecast.from_date)
- & (Coalesce(move.effective_date, move.planned_date)
- <= forecast.to_date)
- & ((move.origin == Null)
- | ~move.origin.like('stock.forecast.line,%')),
- group_by=move.product))
- for product_id, quantity in cursor:
- line = product2line[product_id]
- result[line.id] = Uom.compute_qty(
- line.product.default_uom, quantity, line.unit)
+ where = fields.SQL_OPERATORS['in'](
+ move.product, product2line.keys())
+ cursor.execute(*move.join(location_from,
+ condition=move.from_location == location_from.id
+ ).join(location_to,
+ condition=move.to_location == location_to.id
+ ).select(move.product, Sum(move.internal_quantity),
+ where=where
+ & (location_from.left >= forecast.warehouse.left)
+ & (location_from.right <= forecast.warehouse.right)
+ & (location_to.left >= forecast.destination.left)
+ & (location_to.right <= forecast.destination.right)
+ & (move.state != 'cancelled')
+ & (Coalesce(move.effective_date, move.planned_date)
+ >= forecast.from_date)
+ & (Coalesce(move.effective_date, move.planned_date)
+ <= forecast.to_date)
+ & ((move.origin == Null)
+ | ~move.origin.like('stock.forecast.line,%')),
+ group_by=move.product))
+ for product_id, quantity in cursor:
+ line = product2line[product_id]
+ result[line.id] = Uom.compute_qty(
+ line.product.default_uom, quantity, line.unit)
return result
@classmethod
diff --git a/modules/stock_lot/stock.py b/modules/stock_lot/stock.py
index cf951dd34d..f5f2b0268f 100644
--- a/modules/stock_lot/stock.py
+++ b/modules/stock_lot/stock.py
@@ -18,7 +18,6 @@ from trytond.modules.stock import StockMixin
from trytond.modules.stock.exceptions import ShipmentCheckQuantityWarning
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, If, Len
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from trytond.wizard import Button, StateTransition, StateView, Wizard
@@ -81,12 +80,10 @@ class Lot(DeactivableMixin, ModelSQL, ModelView, LotMixin, StockMixin):
def get_quantity(cls, lots, name):
location_ids = Transaction().context.get('locations')
product_ids = list(set(l.product.id for l in lots))
- quantities = defaultdict(float)
- for product_ids in grouped_slice(product_ids):
- quantities.update(cls._get_quantity(lots, name, location_ids,
- grouping=('product', 'lot',),
- grouping_filter=(list(product_ids),)))
- return quantities
+ return cls._get_quantity(
+ lots, name, location_ids,
+ grouping=('product', 'lot',),
+ grouping_filter=(product_ids,))
@classmethod
def search_quantity(cls, name, domain=None):
@@ -131,16 +128,13 @@ class Lot(DeactivableMixin, ModelSQL, ModelView, LotMixin, StockMixin):
transaction = Transaction()
def find_moves(cls, state=None):
- for sub_records in grouped_slice(lots):
- domain = [
- ('lot', 'in', [r.id for r in sub_records])
- ]
- if state:
- domain.append(('state', '=', state))
- moves = Move.search(domain, limit=1, order=[])
- if moves:
- return True
- return False
+ domain = [
+ ('lot', 'in', lots)
+ ]
+ if state:
+ domain.append(('state', '=', state))
+ moves = Move.search(domain, limit=1, order=[])
+ return bool(moves)
super().check_modification(
mode, lots, values=values, external=external)
diff --git a/modules/stock_lot_sled/stock.py b/modules/stock_lot_sled/stock.py
index c2b107d124..cfe1db6bf6 100644
--- a/modules/stock_lot_sled/stock.py
+++ b/modules/stock_lot_sled/stock.py
@@ -11,7 +11,6 @@ from trytond.model.exceptions import AccessError
from trytond.modules.stock.exceptions import PeriodCloseError
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .exceptions import LotExpiredError, LotExpiredWarning
@@ -122,23 +121,21 @@ class Lot(LotSledMixin, metaclass=PoolMeta):
if not periods:
return
period, = periods
- for lots in grouped_slice(lots):
- lot_ids = [l.id for l in lots]
- moves = Move.search([
- ('lot', 'in', lot_ids),
- ['OR', [
- ('effective_date', '=', None),
- ('planned_date', '<=', period.date),
- ],
- ('effective_date', '<=', period.date),
- ]], limit=1)
- if moves:
- move, = moves
- raise AccessError(
- gettext('stock_lot_sled'
- '.msg_lot_modify_expiration_date_period_close',
- lot=move.lot.rec_name,
- move=move.rec_name))
+ moves = Move.search([
+ ('lot', 'in', lots),
+ ['OR', [
+ ('effective_date', '=', None),
+ ('planned_date', '<=', period.date),
+ ],
+ ('effective_date', '<=', period.date),
+ ]], limit=1)
+ if moves:
+ move, = moves
+ raise AccessError(
+ gettext('stock_lot_sled'
+ '.msg_lot_modify_expiration_date_period_close',
+ lot=move.lot.rec_name,
+ move=move.rec_name))
class MoveAddLotsStartLot(LotSledMixin, metaclass=PoolMeta):
diff --git a/modules/stock_shipment_measurements/stock.py b/modules/stock_shipment_measurements/stock.py
index e924e19f13..ec41b10331 100644
--- a/modules/stock_shipment_measurements/stock.py
+++ b/modules/stock_shipment_measurements/stock.py
@@ -13,7 +13,6 @@ from trytond.model import Index, ModelSQL, ModelView, Workflow, fields
from trytond.modules.company.model import CompanyValueMixin
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, Id
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -282,15 +281,9 @@ class MeasurementsMixin(object):
where=table.state.in_(states)))
if shipments is not None:
- where = query.where
- for sub_shipments in grouped_slice(shipments):
- query.where = (
- where
- & fields.SQL_OPERATORS['in'](
- table.id, map(int, sub_shipments)))
- cursor.execute(*query)
- else:
- cursor.execute(*query)
+ query.where &= fields.SQL_OPERATORS['in'](
+ table.id, map(int, shipments))
+ cursor.execute(*query)
if shipments:
cls.write(shipments, {})
@@ -328,17 +321,15 @@ class MeasurementsMixin(object):
weights, volumes = {}, {}
states = cls._measurements_states()
- for sub_shipments in grouped_slice(
- (s for s in shipments
+ query.where = fields.SQL_OPERATORS['in'](
+ table.id, map(int, (s for s in shipments
if s.state not in states
or s.internal_weight is None
- or s.internal_volume is None)):
- query.where = fields.SQL_OPERATORS['in'](
- table.id, [s.id for s in sub_shipments])
- cursor.execute(*query)
- for id_, weight, volume in cursor:
- weights[id_] = weight
- volumes[id_] = volume
+ or s.internal_volume is None)))
+ cursor.execute(*query)
+ for id_, weight, volume in cursor:
+ weights[id_] = weight
+ volumes[id_] = volume
for shipment in shipments:
if 'weight' in names:
diff --git a/modules/stock_supply/purchase_request.py b/modules/stock_supply/purchase_request.py
index 3bf710c881..90b6a8aec7 100644
--- a/modules/stock_supply/purchase_request.py
+++ b/modules/stock_supply/purchase_request.py
@@ -7,7 +7,6 @@ from collections import defaultdict
from trytond.model import ModelSQL, ValueMixin, fields
from trytond.pool import Pool, PoolMeta
from trytond.pyson import TimeDelta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
supply_period = fields.TimeDelta(
@@ -80,60 +79,59 @@ class PurchaseRequest(metaclass=PoolMeta):
# compute requests
new_requests = []
- for (min_date, max_date), dates_products in date2products.items():
- for sub_products in grouped_slice(dates_products):
- sub_products = Product.browse(sub_products)
+ for (min_date, max_date), sub_products in date2products.items():
+ sub_products = Product.browse(sub_products)
+
+ product2ops = {}
+ product2ops_other = {}
+ for product in sub_products:
+ for order_point in product.order_points:
+ if (order_point.company != company
+ or not order_point.warehouse_location):
+ continue
+ if order_point.type == 'purchase':
+ dict_ = product2ops
+ else:
+ dict_ = product2ops_other
+ dict_[
+ (order_point.warehouse_location.id,
+ order_point.product.id)
+ ] = order_point
+
+ product_ids = [p.id for p in sub_products]
+ with Transaction().set_context(
+ forecast=True,
+ stock_date_end=min_date):
+ pbl = Product.products_by_location(warehouse_ids,
+ with_childs=True, grouping_filter=(product_ids,))
+ for warehouse_id in warehouse_ids:
+ min_date_qties = defaultdict(int,
+ ((x, pbl.pop((warehouse_id, x), 0))
+ for x in product_ids))
+ # Do not compute shortage for product
+ # with different order point
+ product_ids = [
+ p.id for p in sub_products
+ if (warehouse_id, p.id) not in product2ops_other]
+ # Search for shortage between min-max
+ shortages = cls.get_shortage(
+ warehouse_id, product_ids, min_date, max_date,
+ min_date_qties=min_date_qties,
+ order_points=product2ops)
- product2ops = {}
- product2ops_other = {}
for product in sub_products:
- for order_point in product.order_points:
- if (order_point.company != company
- or not order_point.warehouse_location):
- continue
- if order_point.type == 'purchase':
- dict_ = product2ops
- else:
- dict_ = product2ops_other
- dict_[
- (order_point.warehouse_location.id,
- order_point.product.id)
- ] = order_point
-
- product_ids = [p.id for p in sub_products]
- with Transaction().set_context(
- forecast=True,
- stock_date_end=min_date):
- pbl = Product.products_by_location(warehouse_ids,
- with_childs=True, grouping_filter=(product_ids,))
- for warehouse_id in warehouse_ids:
- min_date_qties = defaultdict(int,
- ((x, pbl.pop((warehouse_id, x), 0))
- for x in product_ids))
- # Do not compute shortage for product
- # with different order point
- product_ids = [
- p.id for p in sub_products
- if (warehouse_id, p.id) not in product2ops_other]
- # Search for shortage between min-max
- shortages = cls.get_shortage(
- warehouse_id, product_ids, min_date, max_date,
- min_date_qties=min_date_qties,
- order_points=product2ops)
-
- for product in sub_products:
- if product.id not in shortages:
- continue
- shortage_date, product_quantity = shortages[product.id]
- if shortage_date is None or product_quantity is None:
- continue
- order_point = product2ops.get(
- (warehouse_id, product.id))
- # generate request values
- request = cls.compute_request(product,
- warehouse_id, shortage_date, product_quantity,
- company, order_point)
- new_requests.append(request)
+ if product.id not in shortages:
+ continue
+ shortage_date, product_quantity = shortages[product.id]
+ if shortage_date is None or product_quantity is None:
+ continue
+ order_point = product2ops.get(
+ (warehouse_id, product.id))
+ # generate request values
+ request = cls.compute_request(product,
+ warehouse_id, shortage_date, product_quantity,
+ company, order_point)
+ new_requests.append(request)
# delete purchase requests without a purchase line
products = set(products)
diff --git a/modules/stock_supply/shipment.py b/modules/stock_supply/shipment.py
index ad36965cb5..d7e0fce8ca 100644
--- a/modules/stock_supply/shipment.py
+++ b/modules/stock_supply/shipment.py
@@ -5,7 +5,6 @@ from collections import defaultdict
from trytond.model import ModelSQL, ModelView
from trytond.pool import Pool
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -97,11 +96,9 @@ class ShipmentInternal(ModelSQL, ModelView):
product_ids = [p.id for p in products]
pbl = get_pbl(None, today, None)
else:
- product_ids = list(id2product.keys())
- product_ids.sort()
+ product_ids = sorted(id2product.keys())
pbl = defaultdict(int)
- for sub_product_ids in grouped_slice(product_ids):
- pbl.update(get_pbl(None, today, (list(sub_product_ids),)))
+ pbl.update(get_pbl(None, today, (product_ids,)))
shipments = []
date = today
@@ -188,10 +185,9 @@ class ShipmentInternal(ModelSQL, ModelView):
for key, qty in get_pbl(date, date, None).items():
current_qties[key] += qty
else:
- for sub_product_ids in grouped_slice(product_ids):
- for key, qty in get_pbl(
- date, date, (list(sub_product_ids),)).items():
- current_qties[key] += qty
+ for key, qty in get_pbl(
+ date, date, (product_ids,)).items():
+ current_qties[key] += qty
if shipments:
cls.save(shipments)
diff --git a/modules/stock_supply_production/production.py b/modules/stock_supply_production/production.py
index 6fc8822e67..5f226b3595 100644
--- a/modules/stock_supply_production/production.py
+++ b/modules/stock_supply_production/production.py
@@ -6,7 +6,6 @@ from collections import defaultdict
from trytond.model import ModelSQL, ValueMixin, fields
from trytond.pool import Pool, PoolMeta
from trytond.pyson import TimeDelta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
supply_period = fields.TimeDelta(
@@ -89,60 +88,59 @@ class Production(metaclass=PoolMeta):
date2products[min_date, max_date].append(product)
requests = []
- for (min_date, max_date), dates_products in date2products.items():
- for sub_products in grouped_slice(products):
- sub_products = Product.browse(sub_products)
+ for (min_date, max_date), sub_products in date2products.items():
+ sub_products = Product.browse(sub_products)
+
+ product2ops = {}
+ product2ops_other = {}
+ for product in sub_products:
+ for order_point in product.order_points:
+ if (order_point.company != company
+ or not order_point.warehouse_location):
+ continue
+ if order_point.type == 'production':
+ dict_ = product2ops
+ else:
+ dict_ = product2ops_other
+ dict_[
+ (order_point.warehouse_location.id,
+ order_point.product.id)
+ ] = order_point
+
+ product_ids = [p.id for p in sub_products]
+ with Transaction().set_context(
+ forecast=True,
+ stock_date_end=min_date):
+ pbl = Product.products_by_location(
+ warehouse_ids,
+ with_childs=True,
+ grouping_filter=(product_ids,))
+
+ for warehouse in warehouses:
+ min_date_qties = defaultdict(int,
+ ((x, pbl.pop((warehouse.id, x), 0))
+ for x in product_ids))
+ # Do not compute shortage for product
+ # with different order point
+ product_ids = [
+ p.id for p in sub_products
+ if (warehouse.id, p.id) not in product2ops_other]
+ # Search for shortage between min-max
+ shortages = cls.get_shortage(
+ warehouse.id, product_ids, min_date, max_date,
+ min_date_qties=min_date_qties,
+ order_points=product2ops)
- product2ops = {}
- product2ops_other = {}
for product in sub_products:
- for order_point in product.order_points:
- if (order_point.company != company
- or not order_point.warehouse_location):
- continue
- if order_point.type == 'production':
- dict_ = product2ops
- else:
- dict_ = product2ops_other
- dict_[
- (order_point.warehouse_location.id,
- order_point.product.id)
- ] = order_point
-
- product_ids = [p.id for p in sub_products]
- with Transaction().set_context(
- forecast=True,
- stock_date_end=min_date):
- pbl = Product.products_by_location(
- warehouse_ids,
- with_childs=True,
- grouping_filter=(product_ids,))
-
- for warehouse in warehouses:
- min_date_qties = defaultdict(int,
- ((x, pbl.pop((warehouse.id, x), 0))
- for x in product_ids))
- # Do not compute shortage for product
- # with different order point
- product_ids = [
- p.id for p in sub_products
- if (warehouse.id, p.id) not in product2ops_other]
- # Search for shortage between min-max
- shortages = cls.get_shortage(
- warehouse.id, product_ids, min_date, max_date,
- min_date_qties=min_date_qties,
- order_points=product2ops)
-
- for product in sub_products:
- if product.id not in shortages:
- continue
- for date, quantity in shortages[product.id]:
- order_point = product2ops.get(
- (warehouse.id, product.id))
- req = cls.compute_request(product, warehouse,
- quantity, date, company, order_point)
- req.set_planned_start_date()
- requests.append(req)
+ if product.id not in shortages:
+ continue
+ for date, quantity in shortages[product.id]:
+ order_point = product2ops.get(
+ (warehouse.id, product.id))
+ req = cls.compute_request(product, warehouse,
+ quantity, date, company, order_point)
+ req.set_planned_start_date()
+ requests.append(req)
cls.save(requests)
cls.set_moves(requests)
return requests
diff --git a/modules/timesheet/work.py b/modules/timesheet/work.py
index 1cd273e446..b0494cafc7 100644
--- a/modules/timesheet/work.py
+++ b/modules/timesheet/work.py
@@ -2,6 +2,7 @@
# this repository contains the full copyright notices and license terms.
import datetime
+from collections import defaultdict
from sql import Literal
from sql.aggregate import Sum
@@ -10,7 +11,6 @@ from trytond.i18n import gettext
from trytond.model import ModelSQL, ModelStorage, ModelView, Unique, fields
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, If
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .exceptions import CompanyValidationError
@@ -92,8 +92,7 @@ class Work(ModelSQL, ModelView):
table_w = cls.__table__()
line = Line.__table__()
- ids = [w.id for w in works]
- durations = dict.fromkeys(ids, None)
+ durations = defaultdict(lambda: None)
where = Literal(True)
if context.get('from_date'):
where &= line.date >= context['from_date']
@@ -105,16 +104,15 @@ class Work(ModelSQL, ModelView):
query_table = table_w.join(line, 'LEFT',
condition=line.work == table_w.id)
- for sub_ids in grouped_slice(ids):
- cursor.execute(*query_table.select(table_w.id, Sum(line.duration),
- where=where
- & fields.SQL_OPERATORS['in'](table_w.id, sub_ids),
- group_by=table_w.id))
- for work_id, duration in cursor:
- # SQLite uses float for SUM
- if duration and not isinstance(duration, datetime.timedelta):
- duration = datetime.timedelta(seconds=duration)
- durations[work_id] = duration
+ cursor.execute(*query_table.select(table_w.id, Sum(line.duration),
+ where=where
+ & fields.SQL_OPERATORS['in'](table_w.id, map(int, works)),
+ group_by=table_w.id))
+ for work_id, duration in cursor:
+ # SQLite uses float for SUM
+ if duration and not isinstance(duration, datetime.timedelta):
+ duration = datetime.timedelta(seconds=duration)
+ durations[work_id] = duration
return durations
def get_work(self, name):
diff --git a/modules/web_shop_shopify/product.py b/modules/web_shop_shopify/product.py
index 9e5b9909d1..7aca43832e 100644
--- a/modules/web_shop_shopify/product.py
+++ b/modules/web_shop_shopify/product.py
@@ -13,7 +13,7 @@ from trytond.model import Exclude, ModelSQL, ModelView, fields
from trytond.modules.product.exceptions import TemplateValidationError
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Bool, Eval, If
-from trytond.tools import grouped_slice, slugify
+from trytond.tools import slugify
from trytond.transaction import Transaction
from . import graphql
@@ -481,11 +481,9 @@ class Product_TariffCode(IdentifiersUpdateMixin, metaclass=PoolMeta):
elif isinstance(record.product, Category):
categories.add(record.product)
if categories:
- for sub_categories in grouped_slice(list(categories)):
- templates.update(Template.search([
- ('customs_category', 'in',
- [c.id for c in sub_categories]),
- ]))
+ templates.update(Template.search([
+ ('customs_category', 'in', map(int, categories)),
+ ]))
templates = Template.browse(list(templates))
return Template.get_shopify_identifier_to_update(templates)
@@ -579,11 +577,9 @@ class AttributeSet(IdentifiersUpdateMixin, metaclass=PoolMeta):
def get_shopify_identifier_to_update(cls, sets):
pool = Pool()
Template = pool.get('product.template')
- templates = []
- for sub_sets in grouped_slice(sets):
- templates.extend(Template.search([
- ('attribute_set', 'in', [s.id for s in sub_sets]),
- ]))
+ templates = Template.search([
+ ('attribute_set', 'in', sets),
+ ])
return Template.get_shopify_identifier_to_update(templates)
diff --git a/modules/web_shortener/web.py b/modules/web_shortener/web.py
index 0e26cb7c8f..71f2405644 100644
--- a/modules/web_shortener/web.py
+++ b/modules/web_shortener/web.py
@@ -3,6 +3,7 @@
import logging
import string
+from collections import defaultdict
from urllib.parse import quote, urljoin
from sql.aggregate import Count
@@ -10,7 +11,6 @@ from sql.aggregate import Count
import trytond.config as config
from trytond.model import ModelSQL, ModelView, fields
from trytond.pool import Pool
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from trytond.url import http_host
from trytond.wsgi import Base64Converter
@@ -52,13 +52,13 @@ class ShortenedURL(ModelSQL, ModelView):
access = URLAccess.__table__()
cursor = Transaction().connection.cursor()
- counts = {s.id: 0 for s in shortened_urls}
- for sub_ids in grouped_slice(shortened_urls):
- cursor.execute(*access.select(
- access.url, Count(access.id),
- where=fields.SQL_OPERATORS['in'](access.url, sub_ids),
- group_by=[access.url]))
- counts.update(cursor)
+ counts = defaultdict(int)
+ cursor.execute(*access.select(
+ access.url, Count(access.id),
+ where=fields.SQL_OPERATORS['in'](
+ access.url, map(int, shortened_urls)),
+ group_by=[access.url]))
+ counts.update(cursor)
return counts
@classmethod
diff --git a/trytond-gis/trytond_gis/postgis/__init__.py b/trytond-gis/trytond_gis/postgis/__init__.py
index be62f08573..94c339cd48 100644
--- a/trytond-gis/trytond_gis/postgis/__init__.py
+++ b/trytond-gis/trytond_gis/postgis/__init__.py
@@ -1,17 +1,18 @@
# This file is part of trytond_gis. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
-from trytond.backend.postgresql.database import (
- DatabaseDataError, DatabaseIntegrityError, DatabaseOperationalError,
- DatabaseTimeoutError)
+from trytond.backend.postgresql import (
+ MAX_QUERY_PARAMS, DatabaseDataError, DatabaseIntegrityError,
+ DatabaseOperationalError, DatabaseTimeoutError)
from .database import Database
from .table import TableHandler
__all__ = [
Database,
- DatabaseIntegrityError,
DatabaseDataError,
+ DatabaseIntegrityError,
DatabaseOperationalError,
DatabaseTimeoutError,
+ MAX_QUERY_PARAMS,
TableHandler,
]
diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG
index b53127ab52..286770b9dc 100644
--- a/trytond/CHANGELOG
+++ b/trytond/CHANGELOG
@@ -1,3 +1,5 @@
+* Deprecate ``grouped_slice`` without size
+* Replace ``Database.IN_MAX`` by ``backend.MAX_QUERY_PARAMS``
* Deprecate ``reduce_ids`` for ``SQL_OPERATORS['in']``
* Use array for ``in`` operators
* Update to Psycopg 3
diff --git a/trytond/doc/ref/backend.rst b/trytond/doc/ref/backend.rst
index 01e42665b9..f1856e96ca 100644
--- a/trytond/doc/ref/backend.rst
+++ b/trytond/doc/ref/backend.rst
@@ -16,6 +16,10 @@ The backend manages the database connection and schema manipulation.
The name of the backend configured.
+.. attribute:: MAX_QUERY_PARAMS
+
+ The maximum number of parameters supported by the backend.
+
Database
========
diff --git a/trytond/trytond/admin.py b/trytond/trytond/admin.py
index b352af2834..a1bb68f17a 100644
--- a/trytond/trytond/admin.py
+++ b/trytond/trytond/admin.py
@@ -217,14 +217,13 @@ def validate(models, percentage=100):
if not models:
models = sorted([n for n, _ in pool.iterobject()])
ratio = min(100, percentage) / 100
- in_max = Transaction().database.IN_MAX
+ limit = 1_000
for name in models:
logger.info("validate: %s", name)
Model = pool.get(name)
if not issubclass(Model, ModelStorage):
continue
offset = 0
- limit = in_max
while True:
records = Model.search(
[], order=[('id', 'ASC')], offset=offset, limit=limit)
diff --git a/trytond/trytond/backend/__init__.py b/trytond/trytond/backend/__init__.py
index 3db79b28e2..5b7200fad2 100644
--- a/trytond/trytond/backend/__init__.py
+++ b/trytond/trytond/backend/__init__.py
@@ -13,7 +13,7 @@ import trytond.config as config
__all__ = [
'name', 'Database', 'TableHandler',
'DatabaseIntegrityError', 'DatabaseDataError', 'DatabaseOperationalError',
- 'DatabaseTimeoutError']
+ 'DatabaseTimeoutError', 'MAX_QUERY_PARAMS']
name = urllib.parse.urlparse(config.get('database', 'uri', default='')).scheme
@@ -37,3 +37,4 @@ DatabaseDataError = _module.DatabaseDataError
DatabaseOperationalError = _module.DatabaseOperationalError
DatabaseTimeoutError = _module.DatabaseTimeoutError
TableHandler = _module.TableHandler
+MAX_QUERY_PARAMS = _module.MAX_QUERY_PARAMS
diff --git a/trytond/trytond/backend/database.py b/trytond/trytond/backend/database.py
index 3da718c277..81c7be925f 100644
--- a/trytond/trytond/backend/database.py
+++ b/trytond/trytond/backend/database.py
@@ -16,7 +16,6 @@ class DatabaseInterface(object):
Define generic interface for database connection
'''
flavor = None
- IN_MAX = 1000
def __new__(cls, name=''):
return object.__new__(cls)
diff --git a/trytond/trytond/backend/postgresql/__init__.py b/trytond/trytond/backend/postgresql/__init__.py
index 05a01ca5be..3739956543 100644
--- a/trytond/trytond/backend/postgresql/__init__.py
+++ b/trytond/trytond/backend/postgresql/__init__.py
@@ -6,7 +6,14 @@ from .database import (
DatabaseOperationalError, DatabaseTimeoutError)
from .table import TableHandler
+MAX_QUERY_PARAMS = 50_000 # rounded down from 65_535
+
__all__ = [
- Database, TableHandler,
- DatabaseIntegrityError, DatabaseDataError, DatabaseOperationalError,
- DatabaseTimeoutError]
+ Database,
+ DatabaseDataError,
+ DatabaseIntegrityError,
+ DatabaseOperationalError,
+ DatabaseTimeoutError,
+ MAX_QUERY_PARAMS,
+ TableHandler,
+ ]
diff --git a/trytond/trytond/backend/postgresql/database.py b/trytond/trytond/backend/postgresql/database.py
index dd20341179..ce73dcdcad 100644
--- a/trytond/trytond/backend/postgresql/database.py
+++ b/trytond/trytond/backend/postgresql/database.py
@@ -29,7 +29,6 @@ from sql.operators import Any, BinaryOperator, Concat, NotEqual
from trytond import __series__, config
from trytond.backend.database import DatabaseInterface, SQLType
from trytond.sql.operators import RangeOperator
-from trytond.tools import grouped_slice
from .table import index_method
@@ -486,11 +485,11 @@ class Database(DatabaseInterface):
def lock_records(cls, connection, table, ids):
table = Table(table)
cursor = connection.cursor()
- for sub_ids in grouped_slice(ids):
- where = table.id == Any(list(sub_ids))
- query = table.select(
- Literal(1), where=where, for_=For('UPDATE', nowait=True))
- cursor.execute(*query)
+ query = table.select(
+ Literal(1),
+ where=table.id == Any(list(ids)),
+ for_=For('UPDATE', nowait=True))
+ cursor.execute(*query)
def lock_id(self, id, timeout=None):
if not timeout:
diff --git a/trytond/trytond/backend/sqlite/__init__.py b/trytond/trytond/backend/sqlite/__init__.py
index ac1515602a..3df29910f2 100644
--- a/trytond/trytond/backend/sqlite/__init__.py
+++ b/trytond/trytond/backend/sqlite/__init__.py
@@ -9,10 +9,17 @@ from .database import (
DatabaseOperationalError, DatabaseTimeoutError)
from .table import TableHandler
+MAX_QUERY_PARAMS = 200 # estimation from the SQLITE_MAX_EXPR_DEPTH=1_000
+
if sqlite3.sqlite_version_info < (3, 30, 0):
sys.exit("Tryton sqlite backend requires version 3.30.0 or higher")
__all__ = [
- Database, TableHandler,
- DatabaseIntegrityError, DatabaseDataError, DatabaseOperationalError,
- DatabaseTimeoutError]
+ Database,
+ DatabaseDataError,
+ DatabaseIntegrityError,
+ DatabaseOperationalError,
+ DatabaseTimeoutError,
+ MAX_QUERY_PARAMS,
+ TableHandler,
+ ]
diff --git a/trytond/trytond/backend/sqlite/database.py b/trytond/trytond/backend/sqlite/database.py
index 3e546de9ed..3e0f16e855 100644
--- a/trytond/trytond/backend/sqlite/database.py
+++ b/trytond/trytond/backend/sqlite/database.py
@@ -444,7 +444,6 @@ class Database(DatabaseInterface):
flavor = Flavor(
paramstyle='qmark', function_mapping=MAPPING,
max_limit=-1, filter_=True)
- IN_MAX = 200
TYPES_MAPPING = {
'BIGINT': SQLType('INTEGER', 'INTEGER'),
diff --git a/trytond/trytond/ir/cron.py b/trytond/trytond/ir/cron.py
index 8d29f98b1f..13dd67c162 100644
--- a/trytond/trytond/ir/cron.py
+++ b/trytond/trytond/ir/cron.py
@@ -22,7 +22,6 @@ from trytond.model import (
from trytond.pool import Pool
from trytond.pyson import Eval
from trytond.status import processing
-from trytond.tools import grouped_slice
from trytond.tools import timezone as tz
from trytond.transaction import Transaction, TransactionError
from trytond.worker import run_task
@@ -139,16 +138,15 @@ class Cron(DeactivableMixin, ModelSQL, ModelView):
with transaction.new_transaction() as transaction:
cursor = transaction.connection.cursor()
For = database.get_select_for_skip_locked()
- for sub_crons in grouped_slice(crons):
- ids = [c.id for c in sub_crons]
- query = table.select(
- table.id,
- where=fields.SQL_OPERATORS['in'](table.id, ids),
- for_=For('UPDATE'))
- cursor.execute(*query)
- not_running = {i for i, in cursor}
- running.update(
- (i, True) for i in ids if i not in not_running)
+ ids = [c.id for c in crons]
+ query = table.select(
+ table.id,
+ where=fields.SQL_OPERATORS['in'](table.id, ids),
+ for_=For('UPDATE'))
+ cursor.execute(*query)
+ not_running = {i for i, in cursor}
+ running.update(
+ (i, True) for i in ids if i not in not_running)
return running
@classmethod
diff --git a/trytond/trytond/ir/message.py b/trytond/trytond/ir/message.py
index f1b67caa61..944b003a86 100644
--- a/trytond/trytond/ir/message.py
+++ b/trytond/trytond/ir/message.py
@@ -6,7 +6,6 @@ from collections import defaultdict
from trytond.cache import Cache
from trytond.model import ModelSQL, ModelView, fields
from trytond.pool import Pool
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .lang import get_parent_language as get_parent
@@ -111,22 +110,20 @@ class Message(ModelSQL, ModelView):
id2translations = {}
other_translations = defaultdict(list)
- for sub_messages in grouped_slice(messages):
- res_ids = [m.id for m in sub_messages]
- translations = Translation.search([
- ('lang', '=', INTERNAL_LANG),
+ translations = Translation.search([
+ ('lang', '=', INTERNAL_LANG),
+ ('type', '=', 'model'),
+ ('name', '=', f'{cls.__name__},text'),
+ ('res_id', 'in', messages),
+ ])
+ id2translations.update((t.res_id, t) for t in translations)
+ for translation in Translation.search([
+ ('lang', '!=', INTERNAL_LANG),
('type', '=', 'model'),
('name', '=', f'{cls.__name__},text'),
- ('res_id', 'in', res_ids),
- ])
- id2translations.update((t.res_id, t) for t in translations)
- for translation in Translation.search([
- ('lang', '!=', INTERNAL_LANG),
- ('type', '=', 'model'),
- ('name', '=', f'{cls.__name__},text'),
- ('res_id', 'in', res_ids),
- ]):
- other_translations[translation.res_id].append(translation)
+ ('res_id', 'in', messages),
+ ]):
+ other_translations[translation.res_id].append(translation)
to_save = []
for message in messages:
diff --git a/trytond/trytond/ir/model.py b/trytond/trytond/ir/model.py
index f579393180..97b6556b42 100644
--- a/trytond/trytond/ir/model.py
+++ b/trytond/trytond/ir/model.py
@@ -24,7 +24,7 @@ from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder
from trytond.pyson import Bool, Eval, PYSONDecoder
from trytond.report import Report
from trytond.rpc import RPC
-from trytond.tools import cursor_dict, grouped_slice, is_instance_method
+from trytond.tools import cursor_dict, is_instance_method
from trytond.tools.string_ import StringMatcher
from trytond.transaction import Transaction, without_check_access
from trytond.wizard import (
@@ -1215,14 +1215,14 @@ class ModelButtonClick(DeactivableMixin, ModelSQL, ModelView):
} for r in records])
clicks = defaultdict(list)
- for records in grouped_slice(records):
- records = cls.search([
- ('button', '=', button.id),
- ('record_id', 'in', [r.id for r in records]),
- ], order=[('record_id', 'ASC')])
- clicks.update(
- (k, list(v)) for k, v in groupby(
- records, key=lambda c: c.record_id))
+ records = cls.search([
+ ('button', '=', button.id),
+ ('record_id', 'in', records),
+ ],
+ order=[('record_id', 'ASC')])
+ clicks.update(
+ (k, list(v)) for k, v in groupby(
+ records, key=lambda c: c.record_id))
return clicks
@classmethod
@@ -1230,13 +1230,11 @@ class ModelButtonClick(DeactivableMixin, ModelSQL, ModelView):
def reset(cls, model, names, records):
assert all(r.__class__.__name__ == model for r in records)
- clicks = []
- for records in grouped_slice(records):
- clicks.extend(cls.search([
- ('button.model.name', '=', model),
- ('button.name', 'in', names),
- ('record_id', 'in', [r.id for r in records]),
- ]))
+ clicks = cls.search([
+ ('button.model.name', '=', model),
+ ('button.name', 'in', names),
+ ('record_id', 'in', records),
+ ])
cls.write(clicks, {
'active': False,
})
@@ -1360,33 +1358,32 @@ class ModelData(
def can_modify(cls, records, values):
for Model, records in groupby(
records, key=lambda r: r.__class__):
- for sub_records in grouped_slice(records):
- id2record = {r.id: r for r in sub_records}
- domain = [
- ('model', '=', Model.__name__),
- ('db_id', 'in', list(id2record.keys())),
- ('noupdate', '=', False),
- ]
- if values is not None:
- domain.append(('field_names', '!=', None))
- data = cls.search(domain, order=[])
- for data in data:
- record = id2record[data.db_id]
- if values is None:
- raise AccessError(
- gettext(
- 'ir.msg_delete_xml_record',
- **Model.__names__(record=record)),
- gettext('ir.msg_base_config_record'))
- else:
- for field in values:
- if field in data.field_names:
- raise AccessError(
- gettext(
- 'ir.msg_write_xml_record',
- **cls.__names__(
- field=field, record=record)),
- gettext('ir.msg_base_config_record'))
+ id2record = {r.id: r for r in records}
+ domain = [
+ ('model', '=', Model.__name__),
+ ('db_id', 'in', list(id2record.keys())),
+ ('noupdate', '=', False),
+ ]
+ if values is not None:
+ domain.append(('field_names', '!=', None))
+ data = cls.search(domain, order=[])
+ for data in data:
+ record = id2record[data.db_id]
+ if values is None:
+ raise AccessError(
+ gettext(
+ 'ir.msg_delete_xml_record',
+ **Model.__names__(record=record)),
+ gettext('ir.msg_base_config_record'))
+ else:
+ for field in values:
+ if field in data.field_names:
+ raise AccessError(
+ gettext(
+ 'ir.msg_write_xml_record',
+ **cls.__names__(
+ field=field, record=record)),
+ gettext('ir.msg_base_config_record'))
@classmethod
@without_check_access
@@ -1394,13 +1391,12 @@ class ModelData(
data = []
for name, records in groupby(
records, key=lambda r: r.__class__.__name__):
- for sub_records in grouped_slice(records):
- ids = [r.id for r in sub_records]
- data += cls.search([
- ('model', '=', name),
- ('db_id', 'in', ids),
- ('noupdate', '=', True),
- ], order=[])
+ data += cls.search([
+ ('model', '=', name),
+ ('db_id', 'in', records),
+ ('noupdate', '=', True),
+ ],
+ order=[])
cls.write(data, {'db_id': None})
@classmethod
diff --git a/trytond/trytond/ir/module.py b/trytond/trytond/ir/module.py
index fcf0558e86..741d2cc80b 100644
--- a/trytond/trytond/ir/module.py
+++ b/trytond/trytond/ir/module.py
@@ -13,7 +13,6 @@ from trytond.modules import get_module_info, get_modules
from trytond.pool import Pool
from trytond.pyson import Eval
from trytond.rpc import RPC
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from trytond.wizard import (
Button, StateAction, StateTransition, StateView, Wizard)
@@ -249,11 +248,10 @@ class Module(ModelSQL, ModelView):
pool = Pool()
Dependency = pool.get('ir.module.dependency')
module_names = get_modules(with_test=Pool.test)
- for sub_module_names in grouped_slice(module_names):
- cls.delete(cls.search([
- ('state', '!=', 'activated'),
- ('name', 'not in', list(sub_module_names)),
- ]))
+ cls.delete(cls.search([
+ ('state', '!=', 'activated'),
+ ('name', 'not in', module_names),
+ ]))
modules = cls.search([])
name2module = {m.name: m for m in modules}
@@ -303,12 +301,10 @@ class ModuleDependency(ModelSQL, ModelView):
def get_state(cls, dependencies, name):
pool = Pool()
Module = pool.get('ir.module')
- modules = []
names = [d.name for d in dependencies]
- for sub_names in grouped_slice(names):
- modules.extend(Module.search([
- ('name', 'in', list(sub_names)),
- ]))
+ modules = Module.search([
+ ('name', 'in', names),
+ ])
name2state = {m.name: m.state for m in modules}
return {d.id: name2state.get(d.name, 'unknown') for d in dependencies}
diff --git a/trytond/trytond/ir/note.py b/trytond/trytond/ir/note.py
index 197d6429a0..171d5e6bfd 100644
--- a/trytond/trytond/ir/note.py
+++ b/trytond/trytond/ir/note.py
@@ -1,5 +1,6 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
+from collections import defaultdict
from textwrap import TextWrapper
from sql import Null
@@ -9,7 +10,6 @@ from trytond.i18n import lazy_gettext
from trytond.model import ModelSQL, ModelView, fields
from trytond.pool import Pool
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
from .resource import ResourceMixin, resource_copy
@@ -42,7 +42,7 @@ class Note(ResourceMixin, ModelSQL, ModelView):
return '\n'.join(map(wrapper.fill, message.splitlines()))
@classmethod
- def get_unread(cls, ids, name):
+ def get_unread(cls, notes, name):
pool = Pool()
Read = pool.get('ir.note.read')
cursor = Transaction().connection.cursor()
@@ -50,17 +50,16 @@ class Note(ResourceMixin, ModelSQL, ModelView):
table = cls.__table__()
read = Read.__table__()
- unread = {}
- for sub_ids in grouped_slice(ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- query = table.join(read, 'LEFT',
- condition=(table.id == read.note)
- & (read.user == user_id)
- ).select(table.id,
- Case((read.user != Null, False), else_=True),
- where=where)
- cursor.execute(*query)
- unread.update(cursor)
+ unread = defaultdict(bool)
+ query = table.join(read, 'LEFT',
+ condition=(table.id == read.note)
+ & (read.user == user_id)
+ ).select(table.id,
+ Case((read.user != Null, False), else_=True),
+ where=fields.SQL_OPERATORS['in'](
+ table.id, map(int, notes)))
+ cursor.execute(*query)
+ unread.update(cursor)
return unread
@classmethod
@@ -90,12 +89,10 @@ class Note(ResourceMixin, ModelSQL, ModelView):
if not value:
Read.create([{'note': n.id, 'user': user_id} for n in notes])
else:
- reads = []
- for sub_notes in grouped_slice(notes):
- reads += Read.search([
- ('note', 'in', [n.id for n in sub_notes]),
- ('user', '=', user_id),
- ])
+ reads = Read.search([
+ ('note', 'in', notes),
+ ('user', '=', user_id),
+ ])
Read.delete(reads)
@classmethod
diff --git a/trytond/trytond/ir/queue_.py b/trytond/trytond/ir/queue_.py
index 90738c6d6c..2b9a52227c 100644
--- a/trytond/trytond/ir/queue_.py
+++ b/trytond/trytond/ir/queue_.py
@@ -158,12 +158,9 @@ class Queue(ModelSQL):
else:
instances = None
else:
- ids = set()
with inactive_records():
- for sub_ids in grouped_slice(instances):
- records = Model.search([('id', 'in', list(sub_ids))])
- ids.update(map(int, records))
- if ids:
+ records = Model.search([('id', 'in', instances)])
+ if ids := set(map(int, records)):
instances = Model.browse(
[i for i in instances if i in ids])
else:
diff --git a/trytond/trytond/ir/resource.py b/trytond/trytond/ir/resource.py
index b096251d8a..31efac75fa 100644
--- a/trytond/trytond/ir/resource.py
+++ b/trytond/trytond/ir/resource.py
@@ -10,7 +10,6 @@ from trytond.model import (
Index, Model, ModelSQL, ModelStorage, ModelView, fields)
from trytond.pool import Pool
from trytond.pyson import Eval
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction, without_check_access
__all__ = ['ResourceAccessMixin', 'ResourceMixin', 'resource_copy']
@@ -90,10 +89,9 @@ class ResourceAccessMixin(ModelStorage):
record.resource.id)
for RModel, ids in resources.items():
- for sub_ids in grouped_slice(ids):
- allowed.update(RModel.search([
- ('id', 'in', list(sub_ids)),
- ]))
+ allowed.update(RModel.search([
+ ('id', 'in', ids),
+ ]))
records = [
r for r in records
diff --git a/trytond/trytond/ir/rule.py b/trytond/trytond/ir/rule.py
index c28945c58c..b8e0e749d1 100644
--- a/trytond/trytond/ir/rule.py
+++ b/trytond/trytond/ir/rule.py
@@ -10,7 +10,6 @@ from trytond.model import Check, Index, ModelSQL, ModelView, fields
from trytond.model.exceptions import AccessError, ValidationError
from trytond.pool import Pool
from trytond.pyson import Eval, If, PYSONDecoder
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction, inactive_records
@@ -346,18 +345,13 @@ class Rule(ModelSQL, ModelView):
transaction = Transaction()
def test_domain(ids, domain):
- wrong_ids = []
# Use root to prevent infinite recursion
with transaction.set_user(0, set_context=True), inactive_records():
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- records = Model.search([
- ('id', 'in', sub_ids),
- domain,
- ], order=[])
- wrong_ids.extend(
- set(sub_ids).difference(map(int, records)))
- return wrong_ids
+ records = Model.search([
+ ('id', 'in', ids),
+ domain,
+ ], order=[])
+ return list(set(ids).difference(map(int, records)))
domain = cls.domain_get(model_name, mode=mode)
forbidden = test_domain(ids, domain)
diff --git a/trytond/trytond/ir/translation.py b/trytond/trytond/ir/translation.py
index 2f7d11bae9..5e53dab742 100644
--- a/trytond/trytond/ir/translation.py
+++ b/trytond/trytond/ir/translation.py
@@ -23,7 +23,7 @@ from trytond.i18n import gettext
from trytond.model import Index, ModelSQL, ModelView, fields
from trytond.pool import Pool
from trytond.pyson import Eval, PYSONEncoder
-from trytond.tools import cursor_dict, file_open, grouped_slice
+from trytond.tools import cursor_dict, file_open
from trytond.tools.string_ import LazyString, StringPartitioned
from trytond.transaction import (
Transaction, inactive_records, without_check_access)
@@ -493,16 +493,15 @@ class Translation(
fuzzy_clause = []
else:
fuzzy_clause = [('fuzzy', '=', False)]
- for sub_to_fetch in grouped_slice(to_fetch):
- for translation in cls.search([
- ('lang', '=', lang),
- ('type', '=', ttype),
- ('name', '=', name),
- ('value', '!=', ''),
- ('value', '!=', None),
- ('res_id', 'in', list(sub_to_fetch)),
- ] + fuzzy_clause):
- translations[translation.res_id] = translation.value
+ for translation in cls.search([
+ ('lang', '=', lang),
+ ('type', '=', ttype),
+ ('name', '=', name),
+ ('value', '!=', ''),
+ ('value', '!=', None),
+ ('res_id', 'in', to_fetch),
+ ] + fuzzy_clause):
+ translations[translation.res_id] = translation.value
# Don't store fuzzy translation in cache
if not fuzzy_translation:
for res_id in to_fetch:
@@ -519,15 +518,6 @@ class Translation(
ModelFields = pool.get('ir.model.field')
Model = pool.get('ir.model')
Config = pool.get('ir.configuration')
- transaction = Transaction()
- in_max = transaction.database.IN_MAX
-
- if len(ids) > in_max:
- for i in range(0, len(ids), in_max):
- sub_ids = ids[i:i + in_max]
- sub_values = values[i:i + in_max]
- cls.set_ids(name, ttype, lang, sub_ids, sub_values)
- return
model_name, field_name = name.split(',')
if model_name in ('ir.model.field', 'ir.model'):
@@ -649,13 +639,11 @@ class Translation(
@without_check_access
def delete_ids(cls, model, ttype, ids):
"Delete translation for each id"
- translations = []
- for sub_ids in grouped_slice(ids):
- translations += cls.search([
- ('type', '=', ttype),
- ('name', 'like', model + ',%'),
- ('res_id', 'in', list(sub_ids)),
- ])
+ translations = cls.search([
+ ('type', '=', ttype),
+ ('name', 'like', model + ',%'),
+ ('res_id', 'in', ids),
+ ])
cls.delete(translations)
@classmethod
@@ -676,12 +664,6 @@ class Translation(
parent_args = []
parent_langs = []
clauses = []
- transaction = Transaction()
- if len(args) > transaction.database.IN_MAX:
- for sub_args in grouped_slice(args):
- res.update(cls.get_sources(list(sub_args)))
- return res
-
to_cache = []
for name, ttype, lang, source in args:
name = str(name)
@@ -720,9 +702,10 @@ class Translation(
res[(name, ttype, lang, source)] = parent_src[
(name, ttype, parent_lang, source)]
- in_max = transaction.database.IN_MAX // 7
- for sub_clause in grouped_slice(clauses, in_max):
- for translation in cls.search(['OR'] + list(sub_clause)):
+ while clauses:
+ clauses_slice = clauses[:100]
+ clauses = clauses[100:]
+ for translation in cls.search(['OR'] + clauses_slice):
key = (translation.name, translation.type,
translation.lang, translation.src)
if key not in args:
@@ -1524,11 +1507,10 @@ class TranslationClean(Wizard):
with inactive_records():
for model_name, translations in records.items():
Model = pool.get(model_name)
- for sub_ids in grouped_slice(translations.keys()):
- sub_ids = list(sub_ids)
- records = Model.search([('id', 'in', sub_ids)])
- for record_id in set(sub_ids) - set(map(int, records)):
- to_delete.extend(translations[record_id])
+ ids = translations.keys()
+ records = Model.search([('id', 'in', ids)])
+ for record_id in set(ids) - set(map(int, records)):
+ to_delete.extend(translations[record_id])
# skip translation handled in ir.model.data
models_data = ModelData.search([
diff --git a/trytond/trytond/ir/trigger.py b/trytond/trytond/ir/trigger.py
index 4fb661339f..597f5d1cef 100644
--- a/trytond/trytond/ir/trigger.py
+++ b/trytond/trytond/ir/trigger.py
@@ -15,7 +15,6 @@ from trytond.model import (
from trytond.model.exceptions import ValidationError
from trytond.pool import Pool
from trytond.pyson import Eval, If, PYSONDecoder, TimeDelta
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction
@@ -204,20 +203,18 @@ class Trigger(DeactivableMixin, ModelSQL, ModelView):
# Filter on limit_number
if self.limit_number:
new_ids = []
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- where = fields.SQL_OPERATORS['in'](trigger_log.record_id, sub_ids)
- cursor.execute(*trigger_log.select(
- trigger_log.record_id, Count(),
- where=where & (trigger_log.trigger == self.id),
- group_by=trigger_log.record_id))
- number = dict(cursor)
- for record_id in sub_ids:
- if record_id not in number:
- new_ids.append(record_id)
- continue
- if number[record_id] < self.limit_number:
- new_ids.append(record_id)
+ where = fields.SQL_OPERATORS['in'](trigger_log.record_id, ids)
+ cursor.execute(*trigger_log.select(
+ trigger_log.record_id, Count(),
+ where=where & (trigger_log.trigger == self.id),
+ group_by=trigger_log.record_id))
+ number = dict(cursor)
+ for record_id in ids:
+ if record_id not in number:
+ new_ids.append(record_id)
+ continue
+ if number[record_id] < self.limit_number:
+ new_ids.append(record_id)
ids = new_ids
# Filter on minimum_time_delay
@@ -229,25 +226,23 @@ class Trigger(DeactivableMixin, ModelSQL, ModelView):
now, = cursor.fetchone()
if isinstance(now, str):
now = datetime.datetime.fromisoformat(now)
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- where = fields.SQL_OPERATORS['in'](
- trigger_log.record_id, sub_ids)
- cursor.execute(*trigger_log.select(
- trigger_log.record_id, Max(trigger_log.create_date),
- where=where & (trigger_log.trigger == self.id),
- group_by=trigger_log.record_id))
- delay = dict(cursor)
- for record_id in sub_ids:
- if record_id not in delay:
- new_ids.append(record_id)
- continue
- # SQLite return string for MAX
- if isinstance(delay[record_id], str):
- delay[record_id] = datetime.datetime.fromisoformat(
- delay[record_id])
- if now - delay[record_id] >= self.minimum_time_delay:
- new_ids.append(record_id)
+ where = fields.SQL_OPERATORS['in'](
+ trigger_log.record_id, ids)
+ cursor.execute(*trigger_log.select(
+ trigger_log.record_id, Max(trigger_log.create_date),
+ where=where & (trigger_log.trigger == self.id),
+ group_by=trigger_log.record_id))
+ delay = dict(cursor)
+ for record_id in ids:
+ if record_id not in delay:
+ new_ids.append(record_id)
+ continue
+ # SQLite return string for MAX
+ if isinstance(delay[record_id], str):
+ delay[record_id] = datetime.datetime.fromisoformat(
+ delay[record_id])
+ if now - delay[record_id] >= self.minimum_time_delay:
+ new_ids.append(record_id)
ids = new_ids
records = Model.browse(ids)
diff --git a/trytond/trytond/ir/ui/menu.py b/trytond/trytond/ir/ui/menu.py
index 77eec57df6..a56324832a 100644
--- a/trytond/trytond/ir/ui/menu.py
+++ b/trytond/trytond/ir/ui/menu.py
@@ -7,7 +7,6 @@ from trytond.model import (
DeactivableMixin, ModelSQL, ModelView, fields, sequence_ordered, tree)
from trytond.pool import Pool
from trytond.rpc import RPC
-from trytond.tools import grouped_slice
from trytond.transaction import Transaction, inactive_records
@@ -176,12 +175,10 @@ class UIMenu(
menus = list(filter(has_action_access, menus))
- parent_ids = {x.parent.id for x in menus if x.parent}
- parents = set()
- for sub_parent_ids in grouped_slice(parent_ids):
- parents.update(cls.search([
- ('id', 'in', list(sub_parent_ids)),
- ]))
+ parent_ids = list({x.parent.id for x in menus if x.parent})
+ parents = cls.search([
+ ('id', 'in', parent_ids),
+ ])
# Re-browse to avoid side-cache access
menus = cls.browse([x.id for x in menus
if (x.parent and x.parent in parents) or not x.parent])
@@ -223,14 +220,10 @@ class UIMenu(
def set_action(cls, menus, name, value):
pool = Pool()
ActionKeyword = pool.get('ir.action.keyword')
- action_keywords = []
- transaction = Transaction()
- for i in range(0, len(menus), transaction.database.IN_MAX):
- sub_menus = menus[i:i + transaction.database.IN_MAX]
- action_keywords += ActionKeyword.search([
- ('keyword', '=', 'tree_open'),
- ('model', 'in', [str(menu) for menu in sub_menus]),
- ])
+ action_keywords = ActionKeyword.search([
+ ('keyword', '=', 'tree_open'),
+ ('model', 'in', [str(menu) for menu in menus]),
+ ])
if action_keywords:
with Transaction().set_context(_timestamp=False):
ActionKeyword.delete(action_keywords)
diff --git a/trytond/trytond/model/fields/binary.py b/trytond/trytond/model/fields/binary.py
index d2529a775a..5bd5fbefd0 100644
--- a/trytond/trytond/model/fields/binary.py
+++ b/trytond/trytond/model/fields/binary.py
@@ -6,7 +6,7 @@ import logging
from sql import Column, Null
from trytond.filestore import filestore
-from trytond.tools import cached_property, grouped_slice
+from trytond.tools import cached_property
from trytond.transaction import Transaction
from .field import SQL_OPERATORS, Field
@@ -83,19 +83,18 @@ class Binary(Field):
def store_func(id, prefix):
return self.cast(filestore.get(id, prefix=prefix))
- for sub_ids in grouped_slice(ids):
- cursor.execute(*table.select(
- table.id, Column(table, self.file_id),
- where=SQL_OPERATORS['in'](table.id, sub_ids)
- & (Column(table, self.file_id) != Null)
- & (Column(table, self.file_id) != '')))
- for record_id, file_id in cursor:
- try:
- res[record_id] = store_func(file_id, prefix)
- except (IOError, OSError):
- logger.exception(
- "failed to retrieve %r from filestore at %r",
- file_id, prefix)
+ cursor.execute(*table.select(
+ table.id, Column(table, self.file_id),
+ where=SQL_OPERATORS['in'](table.id, ids)
+ & (Column(table, self.file_id) != Null)
+ & (Column(table, self.file_id) != '')))
+ for record_id, file_id in cursor:
+ try:
+ res[record_id] = store_func(file_id, prefix)
+ except (IOError, OSError):
+ logger.exception(
+ "failed to retrieve %r from filestore at %r",
+ file_id, prefix)
for i in values:
if i['id'] in res:
diff --git a/trytond/trytond/model/fields/dict.py b/trytond/trytond/model/fields/dict.py
index 99fcc05bc9..34358081c2 100644
--- a/trytond/trytond/model/fields/dict.py
+++ b/trytond/trytond/model/fields/dict.py
@@ -8,7 +8,6 @@ from sql import Cast, CombiningQuery, Literal, Null, Select, operators
from trytond import backend
from trytond.pool import Pool
from trytond.protocols.jsonrpc import JSONDecoder, JSONEncoder
-from trytond.tools import grouped_slice
from trytond.tools.immutabledict import ImmutableDict
from trytond.transaction import Transaction
@@ -202,11 +201,9 @@ class TranslatedDict(object):
if self.type_ == 'values':
domain = [('type_', '=', 'selection')]
- records = []
- for key_names in grouped_slice(value.keys()):
- records += SchemaModel.search([
- ('name', 'in', key_names),
- ] + domain)
+ records = SchemaModel.search([
+ ('name', 'in', value.keys()),
+ ] + domain)
keys = SchemaModel.get_keys(records)
if self.type_ == 'keys':
diff --git a/trytond/trytond/model/fields/field.py b/trytond/trytond/model/fields/field.py
index 4bb5393f2a..eb7d3944ca 100644
--- a/trytond/trytond/model/fields/field.py
+++ b/trytond/trytond/model/fields/field.py
@@ -806,8 +806,7 @@ class Translated:
language = Transaction().language
read_size = max(1, min(
inst._cache.size_limit,
- inst._local_cache.size_limit,
- inst._transaction.database.IN_MAX))
+ inst._local_cache.size_limit))
index = inst._ids.index(inst.id)
ids = inst._ids[index:index + read_size]
translations = Translation.get_ids(
diff --git a/trytond/trytond/model/fields/fmany2one.py b/trytond/trytond/model/fields/fmany2one.py
index de3f8fd3d3..c69da08e2c 100644
--- a/trytond/trytond/model/fields/fmany2one.py
+++ b/trytond/trytond/model/fields/fmany2one.py
@@ -3,6 +3,7 @@
from sql import Column, Literal
+from trytond import backend
from trytond.pool import Pool
from trytond.tools import grouped_slice
@@ -62,7 +63,8 @@ def fmany2one(
values = set(filter(all,
(tuple(getattr(r, s) for s in sources) for r in records)))
values2id = {}
- for sub_values in grouped_slice(values):
+ for sub_values in grouped_slice(
+ values, backend.MAX_QUERY_PARAMS // len(target_fields)):
domain = ['OR']
for values in sub_values:
domain.append(
diff --git a/trytond/trytond/model/fields/many2many.py b/trytond/trytond/model/fields/many2many.py
index b61be24210..fc88a781bc 100644
--- a/trytond/trytond/model/fields/many2many.py
+++ b/trytond/trytond/model/fields/many2many.py
@@ -2,14 +2,13 @@
# this repository contains the full copyright notices and license terms.
import warnings
from collections import defaultdict
-from itertools import chain
from sql import Literal, Null
from sql.conditionals import Coalesce
from trytond.pool import Pool
from trytond.pyson import PYSONEncoder
-from trytond.tools import cached_property, grouped_slice
+from trytond.tools import cached_property
from trytond.transaction import Transaction
from .field import (
@@ -153,19 +152,15 @@ class Many2Many(Field):
else:
order += self.order
- relations = []
- for sub_ids in grouped_slice(ids):
- if reference_key:
- references = ['%s,%s' % (model.__name__, x) for x in sub_ids]
- clause = [(self.origin, 'in', references)]
- else:
- clause = [(self.origin, 'in', list(sub_ids))]
- clause += [(self.target, '!=', None)]
- if self.filter:
- clause.append((self.target, 'where', self.filter))
- relations.append(
- [r.id for r in Relation.search(clause, order=order)])
- to_read = list(chain(*relations))
+ if reference_key:
+ references = ['%s,%s' % (model.__name__, x) for x in ids]
+ clause = [(self.origin, 'in', references)]
+ else:
+ clause = [(self.origin, 'in', ids)]
+ clause += [(self.target, '!=', None)]
+ if self.filter:
+ clause.append((self.target, 'where', self.filter))
+ to_read = [r.id for r in Relation.search(clause, order=order)]
relations = {t['id']: t
for t in Relation.read(to_read, [self.origin, self.target])}
@@ -229,19 +224,17 @@ class Many2Many(Field):
target_to_delete.extend(Target.browse(target_ids))
def add(ids, target_ids):
- target_ids = list(map(int, target_ids))
if not target_ids:
return
existing_ids = set()
- for sub_ids in grouped_slice(target_ids):
- relations = Relation.search([
- search_clause(ids),
- (self.target, 'in', list(sub_ids)),
- ])
- for relation in relations:
- existing_ids.add((
- getattr(relation, self.origin).id,
- getattr(relation, self.target).id))
+ relations = Relation.search([
+ search_clause(ids),
+ (self.target, 'in', target_ids),
+ ])
+ for relation in relations:
+ existing_ids.add((
+ getattr(relation, self.origin).id,
+ getattr(relation, self.target).id))
for new_id in target_ids:
for record_id in ids:
if (record_id, new_id) in existing_ids:
@@ -252,14 +245,12 @@ class Many2Many(Field):
})
def remove(ids, target_ids):
- target_ids = list(map(int, target_ids))
if not target_ids:
return
- for sub_ids in grouped_slice(target_ids):
- relation_to_delete.extend(Relation.search([
- search_clause(ids),
- (self.target, 'in', list(sub_ids)),
- ]))
+ relation_to_delete.extend(Relation.search([
+ search_clause(ids),
+ (self.target, 'in', target_ids),
+ ]))
def copy(ids, copy_ids, default=None):
copy_ids = list(map(int, copy_ids))
diff --git a/trytond/trytond/model/fields/one2many.py b/trytond/trytond/model/fields/one2many.py
index 0b16c18a9b..4559095fa1 100644
--- a/trytond/trytond/model/fields/one2many.py
+++ b/trytond/trytond/model/fields/one2many.py
@@ -2,7 +2,6 @@
# this repository contains the full copyright notices and license terms.
import warnings
from collections import defaultdict
-from itertools import chain
from sql import Literal
from sql.conditionals import Coalesce
@@ -11,7 +10,7 @@ from sql.operators import Exists
import trytond.config as config
from trytond.pool import Pool
from trytond.pyson import PYSONEncoder
-from trytond.tools import cached_property, grouped_slice
+from trytond.tools import cached_property
from trytond.transaction import Transaction
from .field import (
@@ -156,17 +155,14 @@ class One2Many(Field):
order += self.order
elif Target._order:
order += Target._order
- targets = []
- for sub_ids in grouped_slice(ids):
- if reference_key:
- references = ['%s,%s' % (model.__name__, x) for x in sub_ids]
- clause = [(self.field, 'in', references)]
- else:
- clause = [(self.field, 'in', list(sub_ids))]
- if self.filter:
- clause.append(self.filter)
- targets.append([r.id for r in Target.search(clause, order=order)])
- to_read = list(chain(*targets))
+ if reference_key:
+ references = ['%s,%s' % (model.__name__, x) for x in ids]
+ clause = [(self.field, 'in', references)]
+ else:
+ clause = [(self.field, 'in', ids)]
+ if self.filter:
+ clause.append(self.filter)
+ to_read = [r.id for r in Target.search(clause, order=order)]
targets = {t['id']: t
for t in Target.read(to_read, ['id', self.field])}
@@ -235,7 +231,6 @@ class One2Many(Field):
to_delete.extend(Target.browse(target_ids))
def add(ids, target_ids):
- target_ids = list(map(int, target_ids))
if not target_ids:
return
targets = Target.browse(target_ids)
@@ -249,17 +244,15 @@ class One2Many(Field):
}))
def remove(ids, target_ids):
- target_ids = list(map(int, target_ids))
if not target_ids:
return
- for sub_ids in grouped_slice(target_ids):
- targets = Target.search([
- search_clause(ids),
- ('id', 'in', list(sub_ids)),
- ])
- to_write.extend((targets, {
- self.field: None,
- }))
+ targets = Target.search([
+ search_clause(ids),
+ ('id', 'in', target_ids),
+ ])
+ to_write.extend((targets, {
+ self.field: None,
+ }))
def copy(ids, copy_ids, default=None):
copy_ids = list(map(int, copy_ids))
diff --git a/trytond/trytond/model/modelsql.py b/trytond/trytond/model/modelsql.py
index ad488f3728..20eb9dd967 100644
--- a/trytond/trytond/model/modelsql.py
+++ b/trytond/trytond/model/modelsql.py
@@ -3,7 +3,7 @@
import datetime
from collections import defaultdict
from functools import wraps
-from itertools import chain, groupby, islice, product, repeat
+from itertools import groupby, islice, product, repeat
from sql import (
Asc, Column, Desc, Expression, Literal, Null, NullsFirst, NullsLast, Table,
@@ -753,16 +753,13 @@ class ModelSQL(ModelStorage):
table = cls.__table_history__()
user = User.__table__()
- revisions = []
columns = [
Coalesce(table.write_date, table.create_date), table.id, user.name]
- for sub_ids in grouped_slice(ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*table.join(user, 'LEFT',
- Coalesce(table.write_uid, table.create_uid) == user.id)
- .select(*columns, where=where, group_by=columns))
- revisions.append(cursor.fetchall())
- revisions = list(chain(*revisions))
+ where = fields.SQL_OPERATORS['in'](table.id, ids)
+ cursor.execute(*table.join(user, 'LEFT',
+ Coalesce(table.write_uid, table.create_uid) == user.id)
+ .select(*columns, where=where, group_by=columns))
+ revisions = list(cursor)
revisions.sort(reverse=True)
# SQLite uses char for COALESCE
if revisions and isinstance(revisions[0][0], str):
@@ -796,20 +793,20 @@ class ModelSQL(ModelStorage):
continue
columns.append(Column(table, fname))
hcolumns.append(Column(history, fname))
- for sub_ids in grouped_slice(ids):
- if not deleted:
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*history.insert(hcolumns,
- table.select(*columns, where=where)))
- else:
- if transaction.database.has_multirow_insert():
+ if not deleted:
+ where = fields.SQL_OPERATORS['in'](table.id, ids)
+ cursor.execute(*history.insert(hcolumns,
+ table.select(*columns, where=where)))
+ else:
+ if transaction.database.has_multirow_insert():
+ for sub_ids in grouped_slice(ids, backend.MAX_QUERY_PARAMS):
cursor.execute(*history.insert(hcolumns,
[[id_, CurrentTimestamp(), user]
for id_ in sub_ids]))
- else:
- for id_ in sub_ids:
- cursor.execute(*history.insert(hcolumns,
- [[id_, CurrentTimestamp(), user]]))
+ else:
+ for id_ in ids:
+ cursor.execute(*history.insert(hcolumns,
+ [[id_, CurrentTimestamp(), user]]))
@classmethod
def _restore_history(cls, ids, datetime, _before=False):
@@ -863,61 +860,58 @@ class ModelSQL(ModelStorage):
window=Window([history.id], order_by=h_order)
).as_('__h_rank'))
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- if not _before:
- hwhere = (column_datetime <= datetime)
+ if not _before:
+ hwhere = (column_datetime <= datetime)
+ else:
+ hwhere = (column_datetime < datetime)
+
+ hwhere &= fields.SQL_OPERATORS['in'](history.id, ids)
+ history_select.query.where = hwhere
+
+ cursor.execute(*history_values)
+ for values in cursor.fetchall():
+ if is_deleted(values):
+ to_delete.add(values[id_idx])
else:
- hwhere = (column_datetime < datetime)
+ to_update.add(values[id_idx])
+ to_delete |= (deleted_sub_ids := set(ids) - to_update)
- hwhere &= fields.SQL_OPERATORS['in'](history.id, sub_ids)
- history_select.query.where = hwhere
+ # we need to skip the deleted IDs that are all None history records
+ # because they could fail the UPDATE
+ if to_delete:
+ history_select.query.where &= (
+ fields.SQL_OPERATORS['not in'](
+ history.id, list(deleted_sub_ids)))
- cursor.execute(*history_values)
- for values in cursor.fetchall():
- if is_deleted(values):
- to_delete.add(values[id_idx])
- else:
- to_update.add(values[id_idx])
- to_delete |= (deleted_sub_ids := set(sub_ids) - to_update)
-
- # we need to skip the deleted IDs that are all None history records
- # because they could fail the UPDATE
- if to_delete:
- history_select.query.where &= (
- fields.SQL_OPERATORS['not in'](
- history.id, list(deleted_sub_ids)))
-
- # Some of the sub_ids are not updated because they are not in the
- # table anymore, they should be undeleted from the value in the
- # history table
- to_undelete = set()
- update_query = table.update(
- columns, history_columns,
- from_=[history_values],
- where=history_values.id == table.id)
- if transaction.database.has_returning():
- update_query.returning = [table.id]
- cursor.execute(*update_query)
- to_undelete.update(to_update - set(r[0] for r in cursor))
- else:
- cursor.execute(*table
- .right_join(history_values, history_values.id == table.id)
- .select(history_values.id, where=table.id == Null))
- to_undelete.update(r[0] for r in cursor)
- cursor.execute(*update_query)
- if to_undelete:
- history_select.query.where = (hwhere
- & fields.SQL_OPERATORS['in'](
- history.id, list(to_undelete)))
- cursor.execute(*table.insert(
- columns,
- history_values.select(*history_columns)))
+ # Some of the ids are not updated because they are not in the
+ # table anymore, they should be undeleted from the value in the
+ # history table
+ to_undelete = set()
+ update_query = table.update(
+ columns, history_columns,
+ from_=[history_values],
+ where=history_values.id == table.id)
+ if transaction.database.has_returning():
+ update_query.returning = [table.id]
+ cursor.execute(*update_query)
+ to_undelete.update(to_update - set(r[0] for r in cursor))
+ else:
+ cursor.execute(*table
+ .right_join(history_values, history_values.id == table.id)
+ .select(history_values.id, where=table.id == Null))
+ to_undelete.update(r[0] for r in cursor)
+ cursor.execute(*update_query)
+ if to_undelete:
+ history_select.query.where = (hwhere
+ & fields.SQL_OPERATORS['in'](
+ history.id, list(to_undelete)))
+ cursor.execute(*table.insert(
+ columns,
+ history_values.select(*history_columns)))
if to_delete:
- for sub_ids in grouped_slice(to_delete):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*table.delete(where=where))
+ cursor.execute(*table.delete(
+ where=fields.SQL_OPERATORS['in'](table.id, ids)))
cls._insert_history(list(to_delete), True)
if to_update:
cls._insert_history(list(to_update))
@@ -939,7 +933,7 @@ class ModelSQL(ModelStorage):
table = cls.__table__()
if not transaction.timestamp:
return
- for sub_ids in grouped_slice(ids):
+ for sub_ids in grouped_slice(ids, backend.MAX_QUERY_PARAMS // 3):
where = Or()
for id_ in sub_ids:
try:
@@ -970,7 +964,6 @@ class ModelSQL(ModelStorage):
def create(cls, vlist):
transaction = Transaction()
cursor = transaction.connection.cursor()
- in_max = transaction.database.IN_MAX
pool = Pool()
Translation = pool.get('ir.translation')
@@ -987,7 +980,8 @@ class ModelSQL(ModelStorage):
if transaction.database.has_multirow_insert():
vlist = (
s for s in grouped_slice(
- vlist, in_max // (len(column_names) or 1)))
+ vlist,
+ backend.MAX_QUERY_PARAMS // (len(column_names) + 1)))
else:
vlist = ([v] for v in vlist)
@@ -1188,14 +1182,12 @@ class ModelSQL(ModelStorage):
result = []
table = cls.__table__()
- in_max = transaction.database.IN_MAX
history_order = None
history_clause = None
history_limit = None
if (cls._history
and transaction.context.get('_datetime')
and not cls._is_table_query()):
- in_max = 1
table = cls.__table_history__()
column = Coalesce(table.write_date, table.create_date)
history_clause = (column <= Transaction().context['_datetime'])
@@ -1255,8 +1247,7 @@ class ModelSQL(ModelStorage):
tables, dom_exp = cls.search_domain(
domain, active_test=False, tables=tables)
from_ = convert_from(None, tables)
- for sub_ids in grouped_slice(ids, in_max):
- sub_ids = list(sub_ids)
+ for sub_ids in ([i] for i in ids) if history_clause else [ids]:
red_sql = fields.SQL_OPERATORS['in'](table.id, sub_ids)
where = red_sql
if history_clause:
@@ -1517,26 +1508,25 @@ class ModelSQL(ModelStorage):
columns.append(Column(table, fname))
update_values.append(field.sql_format(value))
- for sub_ids in grouped_slice(ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- try:
- cursor.execute(*table.update(
- columns, update_values, where=where))
- except (
- backend.DatabaseIntegrityError,
- backend.DatabaseDataError) as exception:
- transaction = Transaction()
- with Transaction().new_transaction():
- if isinstance(
- exception, backend.DatabaseIntegrityError):
- cls.__raise_integrity_error(
- exception, values, list(values.keys()),
- transaction=transaction)
- elif isinstance(exception, backend.DatabaseDataError):
- cls.__raise_data_error(
- exception, values, list(values.keys()),
- transaction=transaction)
- raise
+ try:
+ cursor.execute(*table.update(
+ columns, update_values,
+ where=fields.SQL_OPERATORS['in'](table.id, ids)))
+ except (
+ backend.DatabaseIntegrityError,
+ backend.DatabaseDataError) as exception:
+ transaction = Transaction()
+ with Transaction().new_transaction():
+ if isinstance(
+ exception, backend.DatabaseIntegrityError):
+ cls.__raise_integrity_error(
+ exception, values, list(values.keys()),
+ transaction=transaction)
+ elif isinstance(exception, backend.DatabaseDataError):
+ cls.__raise_data_error(
+ exception, values, list(values.keys()),
+ transaction=transaction)
+ raise
for fname, value in values.items():
field = cls._fields[fname]
@@ -1575,7 +1565,6 @@ class ModelSQL(ModelStorage):
@no_table_query
def delete(cls, records):
transaction = Transaction()
- in_max = transaction.database.IN_MAX
cursor = transaction.connection.cursor()
pool = Pool()
@@ -1590,12 +1579,11 @@ class ModelSQL(ModelStorage):
tree_ids = {}
for fname in cls._mptt_fields:
- tree_ids[fname] = []
- for sub_ids in grouped_slice(ids):
- where = fields.SQL_OPERATORS['in'](
- Column(table, fname), sub_ids)
- cursor.execute(*table.select(table.id, where=where))
- tree_ids[fname] += [x[0] for x in cursor]
+ cursor.execute(*table.select(
+ table.id,
+ where=fields.SQL_OPERATORS['in'](
+ Column(table, fname), ids)))
+ tree_ids[fname] = [x[0] for x in cursor]
foreign_keys_tocheck = []
foreign_keys_toupdate = []
@@ -1617,73 +1605,65 @@ class ModelSQL(ModelStorage):
else:
foreign_keys_tocheck.append((model, field_name))
- if len(records) > in_max:
- # Clean self referencing foreign keys
- # before deleting them by small groups
- # Use the record id as value instead of NULL
- # in case the field is required
- foreign_fields_to_clean = [
- fn for m, fn in foreign_keys_tocheck if m == cls]
- if foreign_fields_to_clean:
- for sub_ids in grouped_slice(ids):
- columns = [
- Column(table, n) for n in foreign_fields_to_clean]
- cursor.execute(*table.update(
- columns, [table.id] * len(foreign_fields_to_clean),
- where=fields.SQL_OPERATORS['in'](table.id, sub_ids)
- ))
+ # Clean self referencing foreign keys
+ # before deleting them by small groups
+ # Use the record id as value instead of NULL
+ # in case the field is required
+ foreign_fields_to_clean = [
+ fn for m, fn in foreign_keys_tocheck if m == cls]
+ if foreign_fields_to_clean:
+ columns = [
+ Column(table, n) for n in foreign_fields_to_clean]
+ cursor.execute(*table.update(
+ columns, [table.id] * len(foreign_fields_to_clean),
+ where=fields.SQL_OPERATORS['in'](table.id, ids)
+ ))
- def get_related_records(Model, field_name, sub_ids):
+ def get_related_records(Model, field_name, ids):
if issubclass(Model, ModelSQL):
foreign_table = Model.__table__()
foreign_red_sql = fields.SQL_OPERATORS['in'](
- Column(foreign_table, field_name), sub_ids)
+ Column(foreign_table, field_name), ids)
cursor.execute(*foreign_table.select(foreign_table.id,
where=foreign_red_sql))
related_records = Model.browse([x[0] for x in cursor])
else:
with without_check_access(), inactive_records():
related_records = Model.search(
- [(field_name, 'in', sub_ids)],
+ [(field_name, 'in', ids)],
order=[])
if Model == cls:
related_records = list(set(related_records) - set(records))
return related_records
- for sub_ids, sub_records in zip(
- grouped_slice(ids), grouped_slice(records)):
- sub_ids = list(sub_ids)
- red_sql = fields.SQL_OPERATORS['in'](table.id, sub_ids)
+ for Model, field_name in foreign_keys_toupdate:
+ related_records = get_related_records(Model, field_name, ids)
+ if related_records:
+ Model.write(related_records, {
+ field_name: None,
+ })
- for Model, field_name in foreign_keys_toupdate:
- related_records = get_related_records(
- Model, field_name, sub_ids)
- if related_records:
- Model.write(related_records, {
- field_name: None,
- })
+ for Model, field_name in foreign_keys_todelete:
+ related_records = get_related_records(Model, field_name, ids)
+ if related_records:
+ Model.delete(related_records)
- for Model, field_name in foreign_keys_todelete:
- related_records = get_related_records(
- Model, field_name, sub_ids)
- if related_records:
- Model.delete(related_records)
+ for Model, field_name in foreign_keys_tocheck:
+ if get_related_records(Model, field_name, ids):
+ error_args = Model.__names__(field_name)
+ raise ForeignKeyError(
+ gettext('ir.msg_foreign_model_exist',
+ **error_args))
- for Model, field_name in foreign_keys_tocheck:
- if get_related_records(Model, field_name, sub_ids):
- error_args = Model.__names__(field_name)
- raise ForeignKeyError(
- gettext('ir.msg_foreign_model_exist',
- **error_args))
-
- try:
- cursor.execute(*table.delete(where=red_sql))
- except backend.DatabaseIntegrityError as exception:
- transaction = Transaction()
- with Transaction().new_transaction():
- cls.__raise_integrity_error(
- exception, {}, transaction=transaction)
- raise
+ try:
+ cursor.execute(*table.delete(
+ where=fields.SQL_OPERATORS['in'](table.id, ids)))
+ except backend.DatabaseIntegrityError as exception:
+ transaction = Transaction()
+ with Transaction().new_transaction():
+ cls.__raise_integrity_error(
+ exception, {}, transaction=transaction)
+ raise
cls._insert_history(ids, deleted=True)
@@ -1704,14 +1684,12 @@ class ModelSQL(ModelStorage):
User = Group = None
table = cls.__table__()
transaction = Transaction()
- in_max = transaction.database.IN_MAX
history_clause = None
limit = None
if (mode == 'read'
and cls._history
and transaction.context.get('_datetime')
and not cls._is_table_query()):
- in_max = 1
table = cls.__table_history__()
column = Coalesce(table.write_date, table.create_date)
history_clause = (column <= Transaction().context['_datetime'])
@@ -1726,7 +1704,7 @@ class ModelSQL(ModelStorage):
tables, dom_exp = cls.search_domain(
domain, active_test=False, tables=tables)
from_ = convert_from(None, tables)
- for sub_ids in grouped_slice(ids, in_max):
+ for sub_ids in ([i] for i in ids) if history_clause else [ids]:
sub_ids = set(sub_ids)
where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
if history_clause:
@@ -1988,30 +1966,26 @@ class ModelSQL(ModelStorage):
return select
cursor.execute(*select)
- rows = list(cursor_dict(cursor, transaction.database.IN_MAX))
+ rows = list(cursor_dict(cursor))
cache = transaction.get_cache()
delete_records = transaction.delete_records[cls.__name__]
- # Can not cache the history value if we are not sure to have fetch all
- # the rows for each records
- if (not (cls._history and transaction.context.get('_datetime'))
- or len(rows) < transaction.database.IN_MAX):
- keys = None
- for data in islice(rows, 0, cache.size_limit):
- if data['id'] in delete_records:
- continue
- if keys is None:
- keys = list(data.keys())
- for k in keys[:]:
- if k in ('_timestamp', '_datetime', '__id'):
- continue
- field = cls._fields[k]
- if not getattr(field, 'datetime_field', None):
- keys.remove(k)
- continue
- for k in keys:
- del data[k]
- cache[cls.__name__][data['id']]._update(data)
+ keys = None
+ for data in islice(rows, 0, cache.size_limit):
+ if data['id'] in delete_records:
+ continue
+ if keys is None:
+ keys = list(data.keys())
+ for k in keys[:]:
+ if k in ('_timestamp', '_datetime', '__id'):
+ continue
+ field = cls._fields[k]
+ if not getattr(field, 'datetime_field', None):
+ keys.remove(k)
+ continue
+ for k in keys:
+ del data[k]
+ cache[cls.__name__][data['id']]._update(data)
return cls.browse([x['id'] for x in rows])
@@ -2115,10 +2089,9 @@ class ModelSQL(ModelStorage):
[Concat(Concat(Coalesce(
parent.select(parent.path,
where=parent.id == parent_column),
- ''), table.id), '/')])
- for sub_ids in grouped_slice(ids):
- query.where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*query)
+ ''), table.id), '/')],
+ where=fields.SQL_OPERATORS['in'](table.id, ids))
+ cursor.execute(*query)
@classmethod
def _update_path(cls, field_names, list_ids):
@@ -2128,9 +2101,9 @@ class ModelSQL(ModelStorage):
table = cls.__table__()
parent = cls.__table__()
- def update_path(query, column, sub_ids):
+ def update_path(query, column, ids):
updated = set()
- query.where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
+ query.where = fields.SQL_OPERATORS['in'](table.id, ids)
cursor.execute(*query)
for old_path, new_path in cursor:
if old_path == new_path:
@@ -2156,10 +2129,8 @@ class ModelSQL(ModelStorage):
.select(path_column,
Concat(Concat(
Coalesce(parent_path_column, ''), table.id), '/')))
- for sub_ids in grouped_slice(ids):
- sub_ids = list(sub_ids)
- while not update_path(query, path_column, sub_ids):
- pass
+ while not update_path(query, path_column, ids):
+ pass
@classmethod
def _update_mptt(cls, field_names, list_ids, values=None):
@@ -2294,50 +2265,47 @@ class ModelSQL(ModelStorage):
else:
columns = list(sql.columns)
columns.insert(0, table.id)
- in_max = transaction.database.IN_MAX // (len(columns) + 1)
- for sub_ids in grouped_slice(ids, in_max):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- if isinstance(sql, Exclude) and sql.where:
- where &= sql.where
+ where = fields.SQL_OPERATORS['in'](table.id, ids)
+ if isinstance(sql, Exclude) and sql.where:
+ where &= sql.where
- cursor.execute(*table.select(*columns, where=where))
+ cursor.execute(*table.select(*columns, where=where))
- where = Literal(False)
- for row in cursor:
- row = list(row)
- clause = table.id != row.pop(0)
- if not database.has_range():
- values = []
- for col in sql.columns:
- if isinstance(col, Range):
- range_ = col.__class__(
- row.pop(0), row.pop(0), row.pop(0))
- values.append(range_)
- else:
- values.append(row.pop(0))
- else:
- values = row
- for column, operator, value in zip(
- sql.columns, sql.operators, values):
- if value is None:
- # NULL is always unique
- clause &= Literal(False)
- clause &= operator(column, value)
- where |= clause
- if isinstance(sql, Exclude) and sql.where:
- where &= sql.where
- cursor.execute(
- *table.select(table.id, where=where, limit=1))
- if cursor.fetchone():
- raise SQLConstraintError(gettext(error))
+ where = Literal(False)
+ for row in cursor:
+ row = list(row)
+ clause = table.id != row.pop(0)
+ if not database.has_range():
+ values = []
+ for col in sql.columns:
+ if isinstance(col, Range):
+ range_ = col.__class__(
+ row.pop(0), row.pop(0), row.pop(0))
+ values.append(range_)
+ else:
+ values.append(row.pop(0))
+ else:
+ values = row
+ for column, operator, value in zip(
+ sql.columns, sql.operators, values):
+ if value is None:
+ # NULL is always unique
+ clause &= Literal(False)
+ clause &= operator(column, value)
+ where |= clause
+ if isinstance(sql, Exclude) and sql.where:
+ where &= sql.where
+ cursor.execute(
+ *table.select(table.id, where=where, limit=1))
+ if cursor.fetchone():
+ raise SQLConstraintError(gettext(error))
elif isinstance(sql, Check):
- for sub_ids in grouped_slice(ids):
- where = fields.SQL_OPERATORS['in'](table.id, sub_ids)
- cursor.execute(*table.select(table.id,
- where=~sql.expression & where,
- limit=1))
- if cursor.fetchone():
- raise SQLConstraintError(gettext(error))
+ cursor.execute(*table.select(table.id,
+ where=~sql.expression
+ & fields.SQL_OPERATORS['in'](table.id, ids),
+ limit=1))
+ if cursor.fetchone():
+ raise SQLConstraintError(gettext(error))
@dualmethod
def lock(cls, records=None):
diff --git a/trytond/trytond/model/modelstorage.py b/trytond/trytond/model/modelstorage.py
index b8069a6001..f77f075bef 100644
--- a/trytond/trytond/model/modelstorage.py
+++ b/trytond/trytond/model/modelstorage.py
@@ -832,12 +832,11 @@ class ModelStorage(Model):
ids = dict.fromkeys(map(int, records))
with without_check_access():
- for sub_records in grouped_slice(records):
- for record in ModelData.search([
- ('model', '=', cls.__name__),
- ('db_id', 'in', [r.id for r in sub_records]),
- ]):
- ids[record.db_id] = f'{record.module}.{record.fs_id}'
+ for record in ModelData.search([
+ ('model', '=', cls.__name__),
+ ('db_id', 'in', records),
+ ]):
+ ids[record.db_id] = f'{record.module}.{record.fs_id}'
return ids
@classmethod
@@ -1453,33 +1452,20 @@ class ModelStorage(Model):
domains[context][domain].append(record)
# Select strategy depending if it is closer to one domain per
# record or one domain for all records
- # Do not use IN_MAX to let spaces for the pyson domain
- in_max = Transaction().database.IN_MAX
- count = in_max // 10
for context, ctx_domains in domains.items():
if (not dict_domain
and len(ctx_domains) > len(records) * 0.5):
- new_domains = {}
- for sub_domains in grouped_slice(
- list(ctx_domains.keys()), count):
- grouped_domain = ['OR']
- grouped_records = []
- for d in sub_domains:
- sub_records = ctx_domains[d]
- grouped_records.extend(sub_records)
- relations = relation_domain(field, sub_records)
- if len(relations) > in_max:
- break
- grouped_domain.append(
- [('id', 'in',
- [r.id for r in relations]), d])
- else:
- new_domains[freeze(grouped_domain)] = \
- grouped_records
- continue
- break
- else:
- domains[context] = new_domains
+ grouped_domain = ['OR']
+ grouped_records = []
+ for d, sub_records in ctx_domains.items():
+ grouped_records.extend(sub_records)
+ relations = relation_domain(field, sub_records)
+ grouped_domain.append(
+ [('id', 'in',
+ [r.id for r in relations]), d])
+ domains[context] = {
+ freeze(grouped_domain): grouped_records,
+ }
else:
domains[freeze(field.context)][freeze(field.domain)].extend(
records)
@@ -1517,68 +1503,67 @@ class ModelStorage(Model):
def validate_relation_domain(field, records, Relation, domain):
relations = relation_domain(field, records)
- if relations:
- for sub_relations in grouped_slice(relations):
- sub_relations = set(sub_relations)
- finds = Relation.search(['AND',
- [('id', 'in', [r.id for r in sub_relations])],
- domain,
- ])
- invalid_relations = sub_relations - set(finds)
- if Relation == cls and field._type not in {
- 'many2one', 'one2many', 'many2many', 'one2one',
- 'reference'}:
- invalid_records = invalid_relations
- elif field._type.endswith('2many'):
- invalid_records = [
- r for r in records
- if invalid_relations.intersection(
- getattr(r, field.name))]
- else:
- invalid_records = [
- r for r in records
- if getattr(r, field.name) in invalid_relations]
- if invalid_records:
- invalid_record = invalid_records.pop()
- if invalid_relations == invalid_records:
- invalid_relation = invalid_record
- else:
- invalid_relation = getattr(
- invalid_record, field.name)
- domain = field.domain
- if is_pyson(domain):
- domain = _record_eval_pyson(records[0], domain)
- if isinstance(domain, dict):
- domain = domain.get(Relation.__name__, [])
- msg = gettext(
- 'ir.msg_domain_validation_record',
- **cls.__names__(field.name, invalid_record))
- fields = set()
- level = 0
- if field not in Relation._fields.values():
- expression = domain_parse(domain)
- for variable in expression.variables:
- parts = variable.split('.')
- fields.add(parts[0])
- level = max(level, len(parts))
- else:
- fields.add(field.name)
- for field_name in sorted(fields):
- env = EvalEnvironment(invalid_relation, Relation)
- invalid_domain = domain_inversion(
- domain, field_name, env)
- if isinstance(invalid_domain, bool):
- continue
- if (len(fields) > 1 # no need to evaluate
- and eval_domain(invalid_domain, env)):
- continue
- field_def = Relation.fields_get(
- [field_name], level=level)
- raise DomainValidationError(
- msg, domain=(invalid_domain, field_def))
- field_def = Relation.fields_get(fields, level=level)
- raise DomainValidationError(
- msg, domain=(domain, field_def))
+ if not relations:
+ return
+ finds = Relation.search(['AND',
+ [('id', 'in', relations)],
+ domain,
+ ])
+ invalid_relations = set(relations) - set(finds)
+ if Relation == cls and field._type not in {
+ 'many2one', 'one2many', 'many2many', 'one2one',
+ 'reference'}:
+ invalid_records = invalid_relations
+ elif field._type.endswith('2many'):
+ invalid_records = [
+ r for r in records
+ if invalid_relations.intersection(
+ getattr(r, field.name))]
+ else:
+ invalid_records = [
+ r for r in records
+ if getattr(r, field.name) in invalid_relations]
+ if invalid_records:
+ invalid_record = invalid_records.pop()
+ if invalid_relations == invalid_records:
+ invalid_relation = invalid_record
+ else:
+ invalid_relation = getattr(
+ invalid_record, field.name)
+ domain = field.domain
+ if is_pyson(domain):
+ domain = _record_eval_pyson(records[0], domain)
+ if isinstance(domain, dict):
+ domain = domain.get(Relation.__name__, [])
+ msg = gettext(
+ 'ir.msg_domain_validation_record',
+ **cls.__names__(field.name, invalid_record))
+ fields = set()
+ level = 0
+ if field not in Relation._fields.values():
+ expression = domain_parse(domain)
+ for variable in expression.variables:
+ parts = variable.split('.')
+ fields.add(parts[0])
+ level = max(level, len(parts))
+ else:
+ fields.add(field.name)
+ for field_name in sorted(fields):
+ env = EvalEnvironment(invalid_relation, Relation)
+ invalid_domain = domain_inversion(
+ domain, field_name, env)
+ if isinstance(invalid_domain, bool):
+ continue
+ if (len(fields) > 1 # no need to evaluate
+ and eval_domain(invalid_domain, env)):
+ continue
+ field_def = Relation.fields_get(
+ [field_name], level=level)
+ raise DomainValidationError(
+ msg, domain=(invalid_domain, field_def))
+ field_def = Relation.fields_get(fields, level=level)
+ raise DomainValidationError(
+ msg, domain=(domain, field_def))
if field_names is None:
field_names = cls._fields.keys()
@@ -1960,8 +1945,7 @@ class ModelStorage(Model):
s.add(id_)
yield id_
read_size = max(1, min(
- self._cache.size_limit, self._local_cache.size_limit,
- self._transaction.database.IN_MAX))
+ self._cache.size_limit, self._local_cache.size_limit))
index = self._ids.index(self.id)
ids = islice(self._ids, index, index + read_size)
ids = unique(filter(filter_, ids))
diff --git a/trytond/trytond/modules/__init__.py b/trytond/trytond/modules/__init__.py
index 88aaba48b3..95f667c832 100644
--- a/trytond/trytond/modules/__init__.py
+++ b/trytond/trytond/modules/__init__.py
@@ -198,10 +198,10 @@ def load_module_graph(graph, pool, update=None, lang=None, indexes=None):
with transaction.connection.cursor() as cursor:
modules = [x.name for x in graph]
module2state = defaultdict(lambda: 'not activated')
- for sub_modules in tools.grouped_slice(modules):
- cursor.execute(*ir_module.select(ir_module.name, ir_module.state,
- where=ir_module.name.in_(list(sub_modules))))
- module2state.update(cursor)
+ cursor.execute(*ir_module.select(
+ ir_module.name, ir_module.state,
+ where=ir_module.name.in_(modules)))
+ module2state.update(cursor)
modules = set(modules)
new_modules = modules - module2state.keys()
diff --git a/trytond/trytond/res/group.py b/trytond/trytond/res/group.py
index f577b47981..f129e952bf 100644
--- a/trytond/trytond/res/group.py
+++ b/trytond/trytond/res/group.py
@@ -7,7 +7,6 @@ from sql import With
from trytond.model import (
DeactivableMixin, ModelSQL, ModelView, Unique, fields, tree)
from trytond.pool import Pool
-from trytond.tools import grouped_slice
class MenuMany2Many(fields.Many2Many):
@@ -17,12 +16,9 @@ class MenuMany2Many(fields.Many2Many):
res = super().get(ids, model, name,
values=values)
menu_ids = list(set(chain(*res.values())))
- test_ids = []
- for sub_ids in grouped_slice(menu_ids):
- test_ids.append(list(map(int, Menu.search([
- ('id', 'in', sub_ids),
- ]))))
- menu_ids = set(chain(*test_ids))
+ menu_ids = list(map(int, Menu.search([
+ ('id', 'in', menu_ids),
+ ])))
for group_id, ids in res.items():
res[group_id] = tuple(id_ for id_ in ids if id_ in menu_ids)
return res
diff --git a/trytond/trytond/res/user.py b/trytond/trytond/res/user.py
index 9d1418cfa6..4b86222146 100644
--- a/trytond/trytond/res/user.py
+++ b/trytond/trytond/res/user.py
@@ -42,7 +42,6 @@ from trytond.pyson import Bool, Eval, PYSONEncoder
from trytond.report import Report, get_email
from trytond.rpc import RPC
from trytond.sendmail import send_message_transactional
-from trytond.tools import grouped_slice
from trytond.tools.email_ import (
EmailNotValidError, normalize_email, set_from_header, validate_email)
from trytond.transaction import Transaction, without_check_access
@@ -319,16 +318,16 @@ class User(avatar_mixin(100, 'login'), DeactivableMixin, ModelSQL, ModelView):
seconds=config.getint('session', 'max_age'))
result = dict((u.id, 0) for u in users)
with without_check_access():
- for sub_ids in grouped_slice(users):
- sessions = Session.search([
- ('create_uid', 'in', sub_ids),
- ], order=[('create_uid', 'ASC')])
+ sessions = Session.search([
+ ('create_uid', 'in', users),
+ ],
+ order=[('create_uid', 'ASC')])
- def filter_(session):
- return abs(session.last_modified_at - now) < timeout
- result.update(dict((i, len(list(g)))
- for i, g in groupby(filter(filter_, sessions),
- attrgetter('create_uid.id'))))
+ def filter_(session):
+ return abs(session.last_modified_at - now) < timeout
+ result.update(dict((i, len(list(g)))
+ for i, g in groupby(filter(filter_, sessions),
+ attrgetter('create_uid.id'))))
return result
@classmethod
@@ -882,10 +881,9 @@ class UserDevice(ModelSQL):
@classmethod
def clear(cls, logins):
- for sub_logins in grouped_slice(logins):
- cls.delete(cls.search([
- ('login', 'in', list(sub_logins)),
- ]))
+ cls.delete(cls.search([
+ ('login', 'in', logins),
+ ]))
class UserAction(ModelSQL):
diff --git a/trytond/trytond/tests/test_history.py b/trytond/trytond/tests/test_history.py
index ce71167c50..98a44e56a8 100644
--- a/trytond/trytond/tests/test_history.py
+++ b/trytond/trytond/tests/test_history.py
@@ -541,54 +541,51 @@ class HistoryTestCase(TestCase):
[l.name for l in history.lines_at_stamp], ['a', 'b'])
@with_transaction()
- def test_search_cursor_max(self):
- 'Test search with number of history entries at database.IN_MAX'
+ def test_search_large_number(self):
+ 'Test search with large number of history entries'
pool = Pool()
History = pool.get('test.history')
transaction = Transaction()
- database = transaction.database
history = History(value=-1)
history.save()
- for history.value in range(database.IN_MAX + 1):
+ for history.value in range(1_000 + 1):
history.save()
with transaction.set_context(_datetime=datetime.datetime.max):
record, = History.search([])
- self.assertEqual(record.value, database.IN_MAX)
+ self.assertEqual(record.value, 1_000)
@with_transaction()
- def test_search_cursor_max_entries(self):
- 'Test search for skipping first history entries at database.IN_MAX'
+ def test_search_large_number_entries(self):
+ 'Test search for skipping first history entries of large number'
pool = Pool()
History = pool.get('test.history')
transaction = Transaction()
- database = transaction.database
for i in range(0, 2):
history = History(value=-1)
history.save()
- for history.value in range(database.IN_MAX + 1):
+ for history.value in range(1_000 + 1):
history.save()
with transaction.set_context(_datetime=datetime.datetime.max):
records = History.search([])
- self.assertEqual({r.value for r in records}, {database.IN_MAX})
+ self.assertEqual({r.value for r in records}, {1_000})
self.assertEqual(len(records), 2)
@with_transaction()
- def test_search_cursor_max_histories(self):
- 'Test search with number of histories at database.IN_MAX'
+ def test_search_large_number_histories(self):
+ 'Test search with large number of histories'
pool = Pool()
History = pool.get('test.history')
transaction = Transaction()
- database = transaction.database
- n = database.IN_MAX + 1
+ n = 1_000
History.create([{'value': 1}] * n)
with transaction.set_context(_datetime=datetime.datetime.max):
diff --git a/trytond/trytond/tests/test_modelsql.py b/trytond/trytond/tests/test_modelsql.py
index 3b9712a2fd..cbc48288c9 100644
--- a/trytond/trytond/tests/test_modelsql.py
+++ b/trytond/trytond/tests/test_modelsql.py
@@ -742,11 +742,6 @@ class ModelSQLTestCase(TestCase):
pool = Pool()
Model = pool.get('test.modelsql.fk.tree')
- # Delete all records at a time
- in_max = Transaction().database.IN_MAX
- self.addCleanup(setattr, Transaction().database, 'IN_MAX', in_max)
- Transaction().database.IN_MAX = 2
-
parent = Model()
parent.save()
child = Model(parent_restrict=parent)
@@ -760,11 +755,6 @@ class ModelSQLTestCase(TestCase):
pool = Pool()
Model = pool.get('test.modelsql.fk.tree')
- # Delete one record at a time
- in_max = Transaction().database.IN_MAX
- self.addCleanup(setattr, Transaction().database, 'IN_MAX', in_max)
- Transaction().database.IN_MAX = 1
-
parent = Model()
parent.save()
child = Model(parent_restrict=parent)
diff --git a/trytond/trytond/tests/test_modelstorage.py b/trytond/trytond/tests/test_modelstorage.py
index bbef489f3f..b025c13b95 100644
--- a/trytond/trytond/tests/test_modelstorage.py
+++ b/trytond/trytond/tests/test_modelstorage.py
@@ -400,29 +400,6 @@ class ModelStorageTestCase(TestCase):
self.assertTrue(cm.exception.domain[0])
self.assertTrue(cm.exception.domain[1]['value'])
- @with_transaction()
- def test_pyson_domain_unique_in_max(self):
- "Test unique pyson domain validation with greater IN_MAX"
- pool = Pool()
- Model = pool.get('test.modelstorage.pyson_domain')
-
- in_max = Transaction().database.IN_MAX
- self.addCleanup(setattr, Transaction().database, 'IN_MAX', in_max)
- Transaction().database.IN_MAX = 1
-
- # Use modulo 6 so len(domains) is greater then len(records) * 0.5
- # and more than 1 (IN_MAX) have the same domain
- Model.create(
- [{'constraint': str(i % 6), 'value': str(i % 6)}
- for i in range(10)])
-
- with self.assertRaises(DomainValidationError) as cm:
- Model.create(
- [{'constraint': str(i % 6), 'value': str(i)}
- for i in range(10)])
- self.assertTrue(cm.exception.domain[0])
- self.assertTrue(cm.exception.domain[1]['value'])
-
@with_transaction()
def test_pyson_domain_single(self):
"Test pyson domain validation for 1 record"
diff --git a/trytond/trytond/tests/test_tools.py b/trytond/trytond/tests/test_tools.py
index a429659933..2e4bceede5 100644
--- a/trytond/trytond/tests/test_tools.py
+++ b/trytond/trytond/tests/test_tools.py
@@ -27,10 +27,10 @@ from trytond.pool import Pool
from trytond.tests.test_tryton import TestCase
from trytond.tools import (
cached_property, decimal_, email_, escape_wildcard, file_open, firstline,
- grouped_slice, is_full_text, is_instance_method, likify, lstrip_wildcard,
- pair, pairwise_longest, reduce_domain, remove_forbidden_chars,
- rstrip_wildcard, slugify, sortable_values, sqlite_apply_types,
- strip_wildcard, timezone, unescape_wildcard, unpair)
+ is_full_text, is_instance_method, likify, lstrip_wildcard, pair,
+ pairwise_longest, reduce_domain, remove_forbidden_chars, rstrip_wildcard,
+ slugify, sortable_values, sqlite_apply_types, strip_wildcard, timezone,
+ unescape_wildcard, unpair)
from trytond.tools.chart import sparkline
from trytond.tools.domain_inversion import (
canonicalize, concat, domain_inversion, eval_domain,
@@ -83,23 +83,6 @@ class ToolsTestCase(TestCase):
self.assertEqual(reduce_domain(i), j,
'%s -> %s != %s' % (i, reduce_domain(i), j))
- def test_grouped_slice(self):
- "Test grouped slice"
- for (values, count, result) in [
- (list(range(10)), 5, [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]),
- (list(range(5)), 5, [[0, 1, 2, 3, 4]]),
- (list(range(5)), 2, [[0, 1], [2, 3], [4]]),
- ]:
- with self.subTest(values=values, count=count):
- self.assertEqual(
- list(map(list, grouped_slice(values, count))), result)
-
- def test_grouped_slice_generator(self):
- "Test grouped slice"
- self.assertEqual(
- list(map(list, grouped_slice((x for x in range(10)), 5))),
- [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]])
-
def test_pairwise_longest(self):
"Test pairwise_longest"
self.assertEqual(
diff --git a/trytond/trytond/tools/misc.py b/trytond/trytond/tools/misc.py
index 24d95afe49..f9c6fda536 100644
--- a/trytond/trytond/tools/misc.py
+++ b/trytond/trytond/tools/misc.py
@@ -149,9 +149,11 @@ def reduce_domain(domain):
def grouped_slice(records, count=None):
'Grouped slice'
- from trytond.transaction import Transaction
if count is None:
- count = Transaction().database.IN_MAX
+ warnings.warn(
+ "grouped_slice without size is deprecated",
+ DeprecationWarning)
+ count = 1
count = max(1, count)
if not isinstance(records, Sized):
records = list(records)