diff --git a/modules/account_statement/CHANGELOG b/modules/account_statement/CHANGELOG index 91c744bdd3..7702c2e6be 100644 --- a/modules/account_statement/CHANGELOG +++ b/modules/account_statement/CHANGELOG @@ -1,3 +1,4 @@ +* Add CSV as import format Version 8.0.0 - 2026-04-20 -------------------------- diff --git a/modules/account_statement/doc/design.rst b/modules/account_statement/doc/design.rst index a5c164d089..9a26deff7a 100644 --- a/modules/account_statement/doc/design.rst +++ b/modules/account_statement/doc/design.rst @@ -105,3 +105,20 @@ Among other things it defines how the statements must be validated. .. |Financial --> Configuration --> Statements --> Statement Journals| replace:: :menuselection:`Financial --> Configuration --> Statements --> Statement Journals` __ https://demo.tryton.org/model/account.statement.journal + +.. _model-account.statement.import.csv: + +CSV Import +========== + +A *CSV Import* defines the format of :abbr:`CSV (Comma-Separated Values)` files +that can be used to `import statements `. + +.. seealso:: + + CSV Import can be found by opening the main menu item: + + |Financial --> Configuration --> Statements --> CSV Imports|__ + + .. |Financial --> Configuration --> Statements --> CSV Imports| replace:: :menuselection:`Financial --> Configuration --> Statements --> CSV Imports` + __ https://demo.tryton.org/model/account.statement.import.csv diff --git a/modules/account_statement/exceptions.py b/modules/account_statement/exceptions.py index 9a74270bc4..be24e6788a 100644 --- a/modules/account_statement/exceptions.py +++ b/modules/account_statement/exceptions.py @@ -1,6 +1,7 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. from trytond.exceptions import UserError, UserWarning +from trytond.model.exceptions import ValidationError class ImportStatementError(UserError): @@ -17,3 +18,7 @@ class StatementValidateWarning(UserWarning): class StatementPostError(UserError): pass + + +class StatementImportCSVValidationError(ValidationError): + pass diff --git a/modules/account_statement/message.xml b/modules/account_statement/message.xml index abd87e20d1..34ff0ae81e 100644 --- a/modules/account_statement/message.xml +++ b/modules/account_statement/message.xml @@ -52,5 +52,8 @@ this repository contains the full copyright notices and license terms. --> The statement file could not be parsed with the following exception: "%(exception)s". + + Invalid date format "%(date_format)s" for the CSV import "%(record)s" with error: "%(error)s" + diff --git a/modules/account_statement/pyproject.toml b/modules/account_statement/pyproject.toml index 0d38586e87..11ea2327d6 100644 --- a/modules/account_statement/pyproject.toml +++ b/modules/account_statement/pyproject.toml @@ -45,6 +45,7 @@ include = [ 'icons/**/*.svg', 'tests/**/*.rst', 'tests/**/*.json', + 'tests/**/*.csv', ] exclude = ['doc'] diff --git a/modules/account_statement/statement.py b/modules/account_statement/statement.py index 7e3315ab5b..1b63dee398 100644 --- a/modules/account_statement/statement.py +++ b/modules/account_statement/statement.py @@ -1,7 +1,12 @@ # This file is part of Tryton. The COPYRIGHT file at the top level of # this repository contains the full copyright notices and license terms. + +import csv +import datetime as dt +import os.path from collections import defaultdict, namedtuple from decimal import Decimal +from io import StringIO from itertools import groupby from sql import Null @@ -25,8 +30,8 @@ from trytond.transaction import Transaction from trytond.wizard import Button, StateAction, StateView, Wizard from .exceptions import ( - ImportStatementError, StatementPostError, StatementValidateError, - StatementValidateWarning) + ImportStatementError, StatementImportCSVValidationError, + StatementPostError, StatementValidateError, StatementValidateWarning) if config.getboolean('account_statement', 'filestore', default=False): file_id = 'origin_file_id' @@ -1264,18 +1269,50 @@ class OriginInformation(DictSchemaMixin, ModelSQL, ModelView): class ImportStatementStart(ModelView): __name__ = 'account.statement.import.start' company = fields.Many2One('company.company', "Company", required=True) - file_ = fields.Binary("File", required=True) - file_format = fields.Selection( - [(None, '')], "File Format", required=True, translate=False) + file_ = fields.Binary("File", required=True, filename='file_name') + file_name = fields.Char("File Name") + file_format = fields.Selection([ + ('csv', "CSV"), + ], "File Format", required=True, translate=False) + csv_format = fields.Many2One( + 'account.statement.import.csv', "CSV Format", + states={ + 'required': Eval('file_format') == 'csv', + 'invisible': Eval('file_format') != 'csv', + }) + csv_journal = fields.Many2One( + 'account.statement.journal', "Journal", + domain=[ + ('validation', '!=', 'balance'), + ], + states={ + 'required': Eval('file_format') == 'csv', + 'invisible': Eval('file_format') != 'csv', + }) @classmethod - def default_file_format(cls): - return None + def __setup__(cls): + super().__setup__() + cls.file_name.states = { + 'required': Eval('file_format').in_(cls._file_name_required()), + } + + @classmethod + def _file_name_required(cls): + return ['csv'] @classmethod def default_company(cls): return Transaction().context.get('company') + @property + def file_name_used(self): + name = self.file_name + while True: + name, ext = os.path.splitext(name) + if not ext: + return name + class ImportStatement(Wizard): __name__ = 'account.statement.import' @@ -1308,6 +1345,233 @@ class ImportStatement(Wizard): action['views'].reverse() return action, data + def parse_csv(self): + csv_format = self.start.csv_format + file_ = StringIO(self.start.file_.decode(csv_format.encoding)) + reader = csv_format.reader(file_) + for _ in range(csv_format.lines_to_skip): + next(reader, None) + statement = self.csv_statement() + origins = [] + count = 0 + for count, row in enumerate(reader, start=1): + origins.extend(self.csv_origin(row, csv_format)) + statement.origins = origins + statement.total_amount = sum(o.amount for o in origins) + statement.number_of_lines = count + yield statement + + def csv_statement(self): + pool = Pool() + Date = pool.get('ir.date') + Statement = pool.get('account.statement') + + statement = Statement() + statement.name = self.start.file_name_used + statement.company = self.start.company + statement.journal = self.start.csv_journal + statement.date = Date.today() + return statement + + def csv_origin(self, row, format_): + pool = Pool() + Origin = pool.get('account.statement.origin') + + origin = Origin() + if format_.number_column is not None: + origin.number = row[format_.number_column] + origin.date = format_.parse_date(row[format_.date_column]) + origin.amount = format_.parse_number(row[format_.amount_column]) + origin.party = self.csv_party(row, format_) + if format_.description_column is not None: + origin.description = row[format_.description_column] + origin.information = self.csv_information(row) + return [origin] + + def csv_party(self, row, format_): + pool = Pool() + Party = pool.get('party.party') + AccountNumber = pool.get('bank.account.number') + + if format_.account_column is not None: + account_number = row[format_.account_column] + if account_number: + numbers = AccountNumber.search(['OR', + ('number', '=', account_number), + ('number_compact', '=', account_number), + ]) + if len(numbers) == 1: + number, = numbers + if number.account.owners: + return number.account.owners[0] + if format_.party_column is not None: + party_name = row[format_.party_column] + if party_name: + parties = Party.search([('rec_name', 'ilike', party_name)]) + if len(parties) == 1: + party, = parties + return party + + def csv_information(self, row): + output = StringIO() + writer = csv.writer( + output, + delimiter=',', + quoting=csv.QUOTE_NONE, + escapechar='\\') + writer.writerow(row) + value = output.getvalue().strip() + return { + 'csv_row': value, + } + + +encodings = ["ascii", "big5", "big5hkscs", "cp037", "cp424", "cp437", "cp500", + "cp720", "cp737", "cp775", "cp850", "cp852", "cp855", "cp856", "cp857", + "cp858", "cp860", "cp861", "cp862", "cp863", "cp864", "cp865", "cp866", + "cp869", "cp874", "cp875", "cp932", "cp949", "cp950", "cp1006", "cp1026", + "cp1140", "cp1250", "cp1251", "cp1252", "cp1253", "cp1254", "cp1255", + "cp1256", "cp1257", "cp1258", "euc_jp", "euc_jis_2004", "euc_jisx0213", + "euc_kr", "gb2312", "gbk", "gb18030", "hz", "iso2022_jp", "iso2022_jp_1", + "iso2022_jp_2", "iso2022_jp_2004", "iso2022_jp_3", "iso2022_jp_ext", + "iso2022_kr", "latin_1", "iso8859_2", "iso8859_3", "iso8859_4", + "iso8859_5", "iso8859_6", "iso8859_7", "iso8859_8", "iso8859_9", + "iso8859_10", "iso8859_13", "iso8859_14", "iso8859_15", "iso8859_16", + "johab", "koi8_r", "koi8_u", "mac_cyrillic", "mac_greek", "mac_iceland", + "mac_latin2", "mac_roman", "mac_turkish", "ptcp154", "shift_jis", + "shift_jis_2004", "shift_jisx0213", "utf_32", "utf_32_be", "utf_32_le", + "utf_16", "utf_16_be", "utf_16_le", "utf_7", "utf_8", "utf_8_sig"] + + +class StatementImportCSV(ModelSQL, ModelView): + __name__ = 'account.statement.import.csv' + + name = fields.Char("Name", required=True) + encoding = fields.Selection( + [(e, e) for e in encodings], + "Encoding", required=True, translate=False) + delimiter = fields.Char("Delimiter", size=1, required=True, strip=False) + quotechar = fields.Char("Quote char", size=1, strip=False) + lines_to_skip = fields.Integer("Lines to Skip", required=True) + date_format = fields.Char( + "Date Format", required=True, strip=False, + help="The format of the date using the 1989 C standard.") + decimal_point = fields.Char( + "Decimal Separator", required=True, strip=False) + thousands_sep = fields.Char("Thousands Separator", strip=False) + + number_column = fields.Integer( + "Number", + domain=['OR', + ('number_column', '=', None), + ('number_column', '>=', 0), + ]) + date_column = fields.Integer( + "Date", required=True, + domain=[('date_column', '>=', 0)]) + amount_column = fields.Integer( + "Amount", required=True, + domain=[('amount_column', '>=', 0)]) + account_column = fields.Integer( + "Account Number", + domain=['OR', + ('account_column', '=', None), + ('account_column', '>=', 0), + ]) + party_column = fields.Integer( + "Party", + domain=['OR', + ('party_column', '=', None), + ('party_column', '>=', 0), + ]) + description_column = fields.Integer( + "Description", + domain=['OR', + ('description_column', '=', None), + ('description_column', '>=', 0), + ]) + + @classmethod + def default_encoding(cls): + return 'utf_8' + + @classmethod + def default_delimiter(cls): + return ',' + + @classmethod + def default_quotechar(cls): + return '"' + + @classmethod + def default_lines_to_skip(cls): + return 0 + + @classmethod + def default_date_format(cls): + pool = Pool() + Lang = pool.get('ir.lang') + lang = Lang.get() + return lang.date + + @classmethod + def default_decimal_point(cls): + pool = Pool() + Lang = pool.get('ir.lang') + lang = Lang.get() + return lang.mon_decimal_point + + @classmethod + def default_thousands_sep(cls): + pool = Pool() + Lang = pool.get('ir.lang') + lang = Lang.get() + return lang.mon_thousands_sep + + def reader(self, csvfile): + return csv.reader(csvfile, **self.fmtparams) + + @property + def fmtparams(self): + return { + 'delimiter': self.delimiter, + 'quotechar': self.quotechar, + } + + @classmethod + def validate_fields(cls, records, field_names): + super().validate_fields(records, field_names) + cls.check_date_format(records, field_names) + + @classmethod + def check_date_format(cls, records, field_names=None): + if field_names and 'date_format' not in field_names: + return + for record in records: + if record.date_format: + try: + dt.datetime.strptime( + dt.datetime.now().strftime(record.date_format), + record.date_format) + except ValueError as e: + raise StatementImportCSVValidationError( + gettext( + 'account_statement' + '.msg_statement_import_csv_invalid_date_format', + date_format=record.date_format, + record=record.rec_name, + error=e)) from e + + def parse_date(self, value): + return dt.datetime.strptime(value, self.date_format).date() + + def parse_number(self, value): + if self.thousands_sep: + value = value.replace(self.thousands_sep, '') + if self.decimal_point: + value = value.replace(self.decimal_point, '.') + return Decimal(value) + class ReconcileStatement(Wizard): __name__ = 'account.statement.reconcile' diff --git a/modules/account_statement/statement.xml b/modules/account_statement/statement.xml index 036e6fa95a..97d98a58d0 100644 --- a/modules/account_statement/statement.xml +++ b/modules/account_statement/statement.xml @@ -314,6 +314,12 @@ this repository contains the full copyright notices and license terms. --> + + csv_row + Row + char + + account.statement.import.start form @@ -346,6 +352,54 @@ this repository contains the full copyright notices and license terms. --> + + account.statement.import.csv + form + statement_import_csv_form + + + + account.statement.import.csv + tree + statement_import_csv_list + + + + CSV Imports + account.statement.import.csv + + + + + + + + + + + + + + + account.statement.import.csv + + + + + + + account.statement.import.csv + + + + + + + Reconcile Statements account.statement.reconcile diff --git a/modules/account_statement/tests/scenario_account_statement_csv.rst b/modules/account_statement/tests/scenario_account_statement_csv.rst new file mode 100644 index 0000000000..cdf2e5ad07 --- /dev/null +++ b/modules/account_statement/tests/scenario_account_statement_csv.rst @@ -0,0 +1,122 @@ +============================== +Account Statement CSV Scenario +============================== + +Imports:: + + >>> from proteus import Model, Wizard + >>> from trytond.modules.account.tests.tools import create_chart, get_accounts + >>> from trytond.modules.company.tests.tools import create_company + >>> from trytond.tests.tools import activate_modules, assertEqual + >>> from trytond.tools import file_open + +Activate modules:: + + >>> config = activate_modules( + ... 'account_statement', + ... create_company, create_chart) + + >>> AccountJournal = Model.get('account.journal') + >>> Bank = Model.get('bank') + >>> BankAccount = Model.get('bank.account') + >>> CSV = Model.get('account.statement.import.csv') + >>> Party = Model.get('party.party') + >>> StatementJournal = Model.get('account.statement.journal') + +Get accounts:: + + >>> accounts = get_accounts() + +Create parties:: + + >>> customer = Party(name="Customer") + >>> customer.save() + >>> bank_party = Party(name="Bank") + >>> bank_party.save() + +Create bank account:: + + >>> bank = Bank() + >>> bank.party = bank_party + >>> bank.save() + >>> bank_account = BankAccount() + >>> bank_account.bank = bank + >>> bank_account.owners.append(customer) + >>> bank_account_number = bank_account.numbers.new() + >>> bank_account_number.type = 'iban' + >>> bank_account_number.number = 'BE47435000000080' + >>> bank_account.save() + +Setup statement journal:: + + >>> account_journal, = AccountJournal.find([('code', '=', 'STA')], limit=1) + >>> journal = StatementJournal( + ... name="Bank", + ... journal=account_journal, + ... account=accounts['cash'], + ... validation='amount') + >>> journal.save() + +Setup CSV import:: + + >>> csv = CSV(name="CSV Bank") + >>> csv.lines_to_skip = 1 + >>> csv.date_format = '%d/%m/%Y' + >>> csv.decimal_point = ',' + >>> csv.thousands_sep = '.' + >>> csv.number_column = 0 + >>> csv.date_column = 1 + >>> csv.amount_column = 2 + >>> csv.account_column = 3 + >>> csv.party_column = 4 + >>> csv.description_column = 5 + >>> csv.save() + +Import CSV file:: + + >>> statement_import = Wizard('account.statement.import') + >>> with file_open('account_statement/tests/statement.csv', mode='rb') as fp: + ... statement_import.form.file_ = fp.read() + >>> statement_import.form.file_name = '001.csv' + >>> statement_import.form.file_format = 'csv' + >>> statement_import.form.csv_format = csv + >>> statement_import.form.csv_journal = journal + >>> statement_import.execute('import_') + +Check statement:: + + >>> (statement,), = statement_import.actions + >>> statement.name + '001' + >>> statement.total_amount + Decimal('980.50') + >>> statement.number_of_lines + 2 + >>> len(statement.origins) + 2 + + >>> origin = statement.origins[0] + >>> origin.number + '0001' + >>> origin.date + datetime.date(2026, 1, 1) + >>> origin.amount + Decimal('1000.50') + >>> assertEqual(origin.party, customer) + >>> origin.description + 'description' + >>> origin.information['csv_row'] + '0001,01/01/2026,1.000\\,50,BE47435000000080,unknown,description' + + >>> origin = statement.origins[1] + >>> origin.number + '0002' + >>> origin.date + datetime.date(2026, 1, 1) + >>> origin.amount + Decimal('-20') + >>> assertEqual(origin.party, customer) + >>> origin.description + '' + >>> origin.information['csv_row'] + '0002,01/01/2026,-20,,Customer,' diff --git a/modules/account_statement/tests/statement.csv b/modules/account_statement/tests/statement.csv new file mode 100644 index 0000000000..ebd9fecee1 --- /dev/null +++ b/modules/account_statement/tests/statement.csv @@ -0,0 +1,3 @@ +number,date,amount,account,party,description +0001,01/01/2026,"1.000,50",BE47435000000080,unknown,description +0002,01/01/2026,-20,,Customer, diff --git a/modules/account_statement/tryton.cfg b/modules/account_statement/tryton.cfg index cffd9bf349..d1cc259121 100644 --- a/modules/account_statement/tryton.cfg +++ b/modules/account_statement/tryton.cfg @@ -26,6 +26,7 @@ model: statement.Origin statement.OriginInformation statement.ImportStatementStart + statement.StatementImportCSV wizard: party.Replace statement.ImportStatement diff --git a/modules/account_statement/view/statement_import_csv_form.xml b/modules/account_statement/view/statement_import_csv_form.xml new file mode 100644 index 0000000000..71a25913fb --- /dev/null +++ b/modules/account_statement/view/statement_import_csv_form.xml @@ -0,0 +1,52 @@ + + +
+