Add CSV as import format for account statement

modules/account_statement/CHANGELOG

@@ -1,3 +1,4 @@
+* Add CSV as import format
 
 Version 8.0.0 - 2026-04-20
 --------------------------

modules/account_statement/doc/design.rst

@@ -105,3 +105,20 @@ Among other things it defines how the statements must be validated.
 
       .. |Financial --> Configuration --> Statements --> Statement Journals| replace:: :menuselection:`Financial --> Configuration --> Statements --> Statement Journals`
       __ https://demo.tryton.org/model/account.statement.journal
+
+.. _model-account.statement.import.csv:
+
+CSV Import
+==========
+
+A *CSV Import* defines the format of :abbr:`CSV (Comma-Separated Values)` files
+that can be used to `import statements <wizard-account.statement.import>`.
+
+.. seealso::
+
+   CSV Import can be found by opening the main menu item:
+
+      |Financial --> Configuration --> Statements --> CSV Imports|__
+
+      .. |Financial --> Configuration --> Statements --> CSV Imports| replace:: :menuselection:`Financial --> Configuration --> Statements --> CSV Imports`
+      __ https://demo.tryton.org/model/account.statement.import.csv

modules/account_statement/exceptions.py

@@ -1,6 +1,7 @@
 # This file is part of Tryton.  The COPYRIGHT file at the top level of
 # this repository contains the full copyright notices and license terms.
 from trytond.exceptions import UserError, UserWarning
+from trytond.model.exceptions import ValidationError
 
 
 class ImportStatementError(UserError):
@@ -17,3 +18,7 @@ class StatementValidateWarning(UserWarning):
 
 class StatementPostError(UserError):
     pass
+
+
+class StatementImportCSVValidationError(ValidationError):
+    pass

modules/account_statement/message.xml

@@ -52,5 +52,8 @@ this repository contains the full copyright notices and license terms. -->
         <record model="ir.message" id="msg_statement_import_error">
             <field name="text">The statement file could not be parsed with the following exception: "%(exception)s".</field>
         </record>
+        <record model="ir.message" id="msg_statement_import_csv_invalid_date_format">
+            <field name="text">Invalid date format "%(date_format)s" for the CSV import "%(record)s" with error: "%(error)s"</field>
+        </record>
     </data>
 </tryton>

modules/account_statement/pyproject.toml

@@ -45,6 +45,7 @@ include = [
     'icons/**/*.svg',
     'tests/**/*.rst',
     'tests/**/*.json',
+    'tests/**/*.csv',
     ]
 exclude = ['doc']
 

modules/account_statement/statement.py

@@ -1,7 +1,12 @@
 # This file is part of Tryton.  The COPYRIGHT file at the top level of
 # this repository contains the full copyright notices and license terms.
+
+import csv
+import datetime as dt
+import os.path
 from collections import defaultdict, namedtuple
 from decimal import Decimal
+from io import StringIO
 from itertools import groupby
 
 from sql import Null
@@ -25,8 +30,8 @@ from trytond.transaction import Transaction
 from trytond.wizard import Button, StateAction, StateView, Wizard
 
 from .exceptions import (
-    ImportStatementError, StatementPostError, StatementValidateError,
-    StatementValidateWarning)
+    ImportStatementError, StatementImportCSVValidationError,
+    StatementPostError, StatementValidateError, StatementValidateWarning)
 
 if config.getboolean('account_statement', 'filestore', default=False):
     file_id = 'origin_file_id'
@@ -1264,18 +1269,50 @@ class OriginInformation(DictSchemaMixin, ModelSQL, ModelView):
 class ImportStatementStart(ModelView):
     __name__ = 'account.statement.import.start'
     company = fields.Many2One('company.company', "Company", required=True)
-    file_ = fields.Binary("File", required=True)
-    file_format = fields.Selection(
-        [(None, '')], "File Format", required=True, translate=False)
+    file_ = fields.Binary("File", required=True, filename='file_name')
+    file_name = fields.Char("File Name")
+    file_format = fields.Selection([
+            ('csv', "CSV"),
+            ], "File Format", required=True, translate=False)
+    csv_format = fields.Many2One(
+        'account.statement.import.csv', "CSV Format",
+        states={
+            'required': Eval('file_format') == 'csv',
+            'invisible': Eval('file_format') != 'csv',
+            })
+    csv_journal = fields.Many2One(
+        'account.statement.journal', "Journal",
+        domain=[
+            ('validation', '!=', 'balance'),
+            ],
+        states={
+            'required': Eval('file_format') == 'csv',
+            'invisible': Eval('file_format') != 'csv',
+            })
 
     @classmethod
-    def default_file_format(cls):
-        return None
+    def __setup__(cls):
+        super().__setup__()
+        cls.file_name.states = {
+            'required': Eval('file_format').in_(cls._file_name_required()),
+            }
+
+    @classmethod
+    def _file_name_required(cls):
+        return ['csv']
 
     @classmethod
     def default_company(cls):
         return Transaction().context.get('company')
 
+    @property
+    def file_name_used(self):
+        name = self.file_name
+        while True:
+            name, ext = os.path.splitext(name)
+            if not ext:
+                return name
+
 
 class ImportStatement(Wizard):
     __name__ = 'account.statement.import'
@@ -1308,6 +1345,233 @@ class ImportStatement(Wizard):
             action['views'].reverse()
         return action, data
 
+    def parse_csv(self):
+        csv_format = self.start.csv_format
+        file_ = StringIO(self.start.file_.decode(csv_format.encoding))
+        reader = csv_format.reader(file_)
+        for _ in range(csv_format.lines_to_skip):
+            next(reader, None)
+        statement = self.csv_statement()
+        origins = []
+        count = 0
+        for count, row in enumerate(reader, start=1):
+            origins.extend(self.csv_origin(row, csv_format))
+        statement.origins = origins
+        statement.total_amount = sum(o.amount for o in origins)
+        statement.number_of_lines = count
+        yield statement
+
+    def csv_statement(self):
+        pool = Pool()
+        Date = pool.get('ir.date')
+        Statement = pool.get('account.statement')
+
+        statement = Statement()
+        statement.name = self.start.file_name_used
+        statement.company = self.start.company
+        statement.journal = self.start.csv_journal
+        statement.date = Date.today()
+        return statement
+
+    def csv_origin(self, row, format_):
+        pool = Pool()
+        Origin = pool.get('account.statement.origin')
+
+        origin = Origin()
+        if format_.number_column is not None:
+            origin.number = row[format_.number_column]
+        origin.date = format_.parse_date(row[format_.date_column])
+        origin.amount = format_.parse_number(row[format_.amount_column])
+        origin.party = self.csv_party(row, format_)
+        if format_.description_column is not None:
+            origin.description = row[format_.description_column]
+        origin.information = self.csv_information(row)
+        return [origin]
+
+    def csv_party(self, row, format_):
+        pool = Pool()
+        Party = pool.get('party.party')
+        AccountNumber = pool.get('bank.account.number')
+
+        if format_.account_column is not None:
+            account_number = row[format_.account_column]
+            if account_number:
+                numbers = AccountNumber.search(['OR',
+                        ('number', '=', account_number),
+                        ('number_compact', '=', account_number),
+                        ])
+                if len(numbers) == 1:
+                    number, = numbers
+                    if number.account.owners:
+                        return number.account.owners[0]
+        if format_.party_column is not None:
+            party_name = row[format_.party_column]
+            if party_name:
+                parties = Party.search([('rec_name', 'ilike', party_name)])
+                if len(parties) == 1:
+                    party, = parties
+                    return party
+
+    def csv_information(self, row):
+        output = StringIO()
+        writer = csv.writer(
+            output,
+            delimiter=',',
+            quoting=csv.QUOTE_NONE,
+            escapechar='\\')
+        writer.writerow(row)
+        value = output.getvalue().strip()
+        return {
+            'csv_row': value,
+            }
+
+
+encodings = ["ascii", "big5", "big5hkscs", "cp037", "cp424", "cp437", "cp500",
+    "cp720", "cp737", "cp775", "cp850", "cp852", "cp855", "cp856", "cp857",
+    "cp858", "cp860", "cp861", "cp862", "cp863", "cp864", "cp865", "cp866",
+    "cp869", "cp874", "cp875", "cp932", "cp949", "cp950", "cp1006", "cp1026",
+    "cp1140", "cp1250", "cp1251", "cp1252", "cp1253", "cp1254", "cp1255",
+    "cp1256", "cp1257", "cp1258", "euc_jp", "euc_jis_2004", "euc_jisx0213",
+    "euc_kr", "gb2312", "gbk", "gb18030", "hz", "iso2022_jp", "iso2022_jp_1",
+    "iso2022_jp_2", "iso2022_jp_2004", "iso2022_jp_3", "iso2022_jp_ext",
+    "iso2022_kr", "latin_1", "iso8859_2", "iso8859_3", "iso8859_4",
+    "iso8859_5", "iso8859_6", "iso8859_7", "iso8859_8", "iso8859_9",
+    "iso8859_10", "iso8859_13", "iso8859_14", "iso8859_15", "iso8859_16",
+    "johab", "koi8_r", "koi8_u", "mac_cyrillic", "mac_greek", "mac_iceland",
+    "mac_latin2", "mac_roman", "mac_turkish", "ptcp154", "shift_jis",
+    "shift_jis_2004", "shift_jisx0213", "utf_32", "utf_32_be", "utf_32_le",
+    "utf_16", "utf_16_be", "utf_16_le", "utf_7", "utf_8", "utf_8_sig"]
+
+
+class StatementImportCSV(ModelSQL, ModelView):
+    __name__ = 'account.statement.import.csv'
+
+    name = fields.Char("Name", required=True)
+    encoding = fields.Selection(
+        [(e, e) for e in encodings],
+        "Encoding", required=True, translate=False)
+    delimiter = fields.Char("Delimiter", size=1, required=True, strip=False)
+    quotechar = fields.Char("Quote char", size=1, strip=False)
+    lines_to_skip = fields.Integer("Lines to Skip", required=True)
+    date_format = fields.Char(
+        "Date Format", required=True, strip=False,
+        help="The format of the date using the 1989 C standard.")
+    decimal_point = fields.Char(
+        "Decimal Separator", required=True, strip=False)
+    thousands_sep = fields.Char("Thousands Separator", strip=False)
+
+    number_column = fields.Integer(
+        "Number",
+        domain=['OR',
+            ('number_column', '=', None),
+            ('number_column', '>=', 0),
+            ])
+    date_column = fields.Integer(
+        "Date", required=True,
+        domain=[('date_column', '>=', 0)])
+    amount_column = fields.Integer(
+        "Amount", required=True,
+        domain=[('amount_column', '>=', 0)])
+    account_column = fields.Integer(
+        "Account Number",
+        domain=['OR',
+            ('account_column', '=', None),
+            ('account_column', '>=', 0),
+            ])
+    party_column = fields.Integer(
+        "Party",
+        domain=['OR',
+            ('party_column', '=', None),
+            ('party_column', '>=', 0),
+            ])
+    description_column = fields.Integer(
+        "Description",
+        domain=['OR',
+            ('description_column', '=', None),
+            ('description_column', '>=', 0),
+            ])
+
+    @classmethod
+    def default_encoding(cls):
+        return 'utf_8'
+
+    @classmethod
+    def default_delimiter(cls):
+        return ','
+
+    @classmethod
+    def default_quotechar(cls):
+        return '"'
+
+    @classmethod
+    def default_lines_to_skip(cls):
+        return 0
+
+    @classmethod
+    def default_date_format(cls):
+        pool = Pool()
+        Lang = pool.get('ir.lang')
+        lang = Lang.get()
+        return lang.date
+
+    @classmethod
+    def default_decimal_point(cls):
+        pool = Pool()
+        Lang = pool.get('ir.lang')
+        lang = Lang.get()
+        return lang.mon_decimal_point
+
+    @classmethod
+    def default_thousands_sep(cls):
+        pool = Pool()
+        Lang = pool.get('ir.lang')
+        lang = Lang.get()
+        return lang.mon_thousands_sep
+
+    def reader(self, csvfile):
+        return csv.reader(csvfile, **self.fmtparams)
+
+    @property
+    def fmtparams(self):
+        return {
+            'delimiter': self.delimiter,
+            'quotechar': self.quotechar,
+            }
+
+    @classmethod
+    def validate_fields(cls, records, field_names):
+        super().validate_fields(records, field_names)
+        cls.check_date_format(records, field_names)
+
+    @classmethod
+    def check_date_format(cls, records, field_names=None):
+        if field_names and 'date_format' not in field_names:
+            return
+        for record in records:
+            if record.date_format:
+                try:
+                    dt.datetime.strptime(
+                        dt.datetime.now().strftime(record.date_format),
+                        record.date_format)
+                except ValueError as e:
+                    raise StatementImportCSVValidationError(
+                        gettext(
+                            'account_statement'
+                            '.msg_statement_import_csv_invalid_date_format',
+                            date_format=record.date_format,
+                            record=record.rec_name,
+                            error=e)) from e
+
+    def parse_date(self, value):
+        return dt.datetime.strptime(value, self.date_format).date()
+
+    def parse_number(self, value):
+        if self.thousands_sep:
+            value = value.replace(self.thousands_sep, '')
+        if self.decimal_point:
+            value = value.replace(self.decimal_point, '.')
+        return Decimal(value)
+
 
 class ReconcileStatement(Wizard):
     __name__ = 'account.statement.reconcile'

modules/account_statement/statement.xml

@@ -314,6 +314,12 @@ this repository contains the full copyright notices and license terms. -->
             <field name="perm_delete" eval="False"/>
         </record>
 
+        <record model="account.statement.origin.information" id="information_csv_row">
+            <field name="name">csv_row</field>
+            <field name="string">Row</field>
+            <field name="type_">char</field>
+        </record>
+
         <record model="ir.ui.view" id="statement_import_start_view_form">
             <field name="model">account.statement.import.start</field>
             <field name="type">form</field>
@@ -346,6 +352,54 @@ this repository contains the full copyright notices and license terms. -->
             <field name="group" ref="group_statement"/>
         </record>
 
+        <record model="ir.ui.view" id="statement_import_csv_view_form">
+            <field name="model">account.statement.import.csv</field>
+            <field name="type">form</field>
+            <field name="name">statement_import_csv_form</field>
+        </record>
+
+        <record model="ir.ui.view" id="statement_import_csv_view_list">
+            <field name="model">account.statement.import.csv</field>
+            <field name="type">tree</field>
+            <field name="name">statement_import_csv_list</field>
+        </record>
+
+        <record model="ir.action.act_window" id="act_statement_import_csv_form">
+            <field name="name">CSV Imports</field>
+            <field name="res_model">account.statement.import.csv</field>
+        </record>
+        <record model="ir.action.act_window.view" id="act_statement_import_csv_form_view1">
+            <field name="sequence" eval="10"/>
+            <field name="view" ref="statement_import_csv_view_list"/>
+            <field name="act_window" ref="act_statement_import_csv_form"/>
+        </record>
+        <record model="ir.action.act_window.view" id="act_statement_import_csv_form_view2">
+            <field name="sequence" eval="20"/>
+            <field name="view" ref="statement_import_csv_view_form"/>
+            <field name="act_window" ref="act_statement_import_csv_form"/>
+        </record>
+        <menuitem
+            parent="menu_statement_configuration"
+            action="act_statement_import_csv_form"
+            sequence="30"
+            id="menu_statement_import_csv_form"/>
+
+        <record model="ir.model.access" id="access_statement_import_csv">
+            <field name="model">account.statement.import.csv</field>
+            <field name="perm_read" eval="True"/>
+            <field name="perm_write" eval="False"/>
+            <field name="perm_create" eval="False"/>
+            <field name="perm_delete" eval="False"/>
+        </record>
+        <record model="ir.model.access" id="access_statement_import_csv_account_admin">
+            <field name="model">account.statement.import.csv</field>
+            <field name="group" ref="account.group_account_admin"/>
+            <field name="perm_read" eval="True"/>
+            <field name="perm_write" eval="True"/>
+            <field name="perm_create" eval="True"/>
+            <field name="perm_delete" eval="True"/>
+        </record>
+
         <record model="ir.action.wizard" id="act_reconcile">
             <field name="name">Reconcile Statements</field>
             <field name="wiz_name">account.statement.reconcile</field>

modules/account_statement/tests/scenario_account_statement_csv.rst

@@ -0,0 +1,122 @@
+==============================
+Account Statement CSV Scenario
+==============================
+
+Imports::
+
+    >>> from proteus import Model, Wizard
+    >>> from trytond.modules.account.tests.tools import create_chart, get_accounts
+    >>> from trytond.modules.company.tests.tools import create_company
+    >>> from trytond.tests.tools import activate_modules, assertEqual
+    >>> from trytond.tools import file_open
+
+Activate modules::
+
+    >>> config = activate_modules(
+    ...     'account_statement',
+    ...     create_company, create_chart)
+
+    >>> AccountJournal = Model.get('account.journal')
+    >>> Bank = Model.get('bank')
+    >>> BankAccount = Model.get('bank.account')
+    >>> CSV = Model.get('account.statement.import.csv')
+    >>> Party = Model.get('party.party')
+    >>> StatementJournal = Model.get('account.statement.journal')
+
+Get accounts::
+
+    >>> accounts = get_accounts()
+
+Create parties::
+
+    >>> customer = Party(name="Customer")
+    >>> customer.save()
+    >>> bank_party = Party(name="Bank")
+    >>> bank_party.save()
+
+Create bank account::
+
+    >>> bank = Bank()
+    >>> bank.party = bank_party
+    >>> bank.save()
+    >>> bank_account = BankAccount()
+    >>> bank_account.bank = bank
+    >>> bank_account.owners.append(customer)
+    >>> bank_account_number = bank_account.numbers.new()
+    >>> bank_account_number.type = 'iban'
+    >>> bank_account_number.number = 'BE47435000000080'
+    >>> bank_account.save()
+
+Setup statement journal::
+
+    >>> account_journal, = AccountJournal.find([('code', '=', 'STA')], limit=1)
+    >>> journal = StatementJournal(
+    ...     name="Bank",
+    ...     journal=account_journal,
+    ...     account=accounts['cash'],
+    ...     validation='amount')
+    >>> journal.save()
+
+Setup CSV import::
+
+    >>> csv = CSV(name="CSV Bank")
+    >>> csv.lines_to_skip = 1
+    >>> csv.date_format = '%d/%m/%Y'
+    >>> csv.decimal_point = ','
+    >>> csv.thousands_sep = '.'
+    >>> csv.number_column = 0
+    >>> csv.date_column = 1
+    >>> csv.amount_column = 2
+    >>> csv.account_column = 3
+    >>> csv.party_column = 4
+    >>> csv.description_column = 5
+    >>> csv.save()
+
+Import CSV file::
+
+    >>> statement_import = Wizard('account.statement.import')
+    >>> with file_open('account_statement/tests/statement.csv', mode='rb') as fp:
+    ...     statement_import.form.file_ = fp.read()
+    >>> statement_import.form.file_name = '001.csv'
+    >>> statement_import.form.file_format = 'csv'
+    >>> statement_import.form.csv_format = csv
+    >>> statement_import.form.csv_journal = journal
+    >>> statement_import.execute('import_')
+
+Check statement::
+
+    >>> (statement,), = statement_import.actions
+    >>> statement.name
+    '001'
+    >>> statement.total_amount
+    Decimal('980.50')
+    >>> statement.number_of_lines
+    2
+    >>> len(statement.origins)
+    2
+
+    >>> origin = statement.origins[0]
+    >>> origin.number
+    '0001'
+    >>> origin.date
+    datetime.date(2026, 1, 1)
+    >>> origin.amount
+    Decimal('1000.50')
+    >>> assertEqual(origin.party, customer)
+    >>> origin.description
+    'description'
+    >>> origin.information['csv_row']
+    '0001,01/01/2026,1.000\\,50,BE47435000000080,unknown,description'
+
+    >>> origin = statement.origins[1]
+    >>> origin.number
+    '0002'
+    >>> origin.date
+    datetime.date(2026, 1, 1)
+    >>> origin.amount
+    Decimal('-20')
+    >>> assertEqual(origin.party, customer)
+    >>> origin.description
+    ''
+    >>> origin.information['csv_row']
+    '0002,01/01/2026,-20,,Customer,'

modules/account_statement/tests/statement.csv

@@ -0,0 +1,3 @@
+number,date,amount,account,party,description
+0001,01/01/2026,"1.000,50",BE47435000000080,unknown,description
+0002,01/01/2026,-20,,Customer,

modules/account_statement/tryton.cfg

@@ -26,6 +26,7 @@ model:
     statement.Origin
     statement.OriginInformation
     statement.ImportStatementStart
+    statement.StatementImportCSV
 wizard:
     party.Replace
     statement.ImportStatement

modules/account_statement/view/statement_import_csv_form.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!-- This file is part of Tryton.  The COPYRIGHT file at the top level of
+this repository contains the full copyright notices and license terms. -->
+<form>
+    <label name="name"/>
+    <field name="name" colspan="3"/>
+
+    <separator string="Format" id="format" colspan="4"/>
+    <label name="encoding"/>
+    <field name="encoding"/>
+    <label name="lines_to_skip"/>
+    <field name="lines_to_skip"/>
+
+    <label name="delimiter"/>
+    <field name="delimiter"/>
+    <label name="quotechar"/>
+    <field name="quotechar"/>
+
+    <label name="date_format"/>
+    <field name="date_format"/>
+    <newline/>
+
+    <label name="decimal_point"/>
+    <field name="decimal_point"/>
+    <label name="thousands_sep"/>
+    <field name="thousands_sep"/>
+
+    <separator string="Columns" id="columns" colspan="4"/>
+    <label name="number_column"/>
+    <field name="number_column"/>
+    <newline/>
+
+    <label name="date_column"/>
+    <field name="date_column"/>
+    <newline/>
+
+    <label name="amount_column"/>
+    <field name="amount_column"/>
+    <newline/>
+
+    <label name="account_column"/>
+    <field name="account_column"/>
+    <newline/>
+
+    <label name="party_column"/>
+    <field name="party_column"/>
+    <newline/>
+
+    <label name="description_column"/>
+    <field name="description_column"/>
+    <newline/>
+</form>

modules/account_statement/view/statement_import_csv_list.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<!-- This file is part of Tryton.  The COPYRIGHT file at the top level of
+this repository contains the full copyright notices and license terms. -->
+<tree>
+    <field name="name" expand="2"/>
+</tree>

modules/account_statement/view/statement_import_start_form.xml

@@ -5,8 +5,17 @@ copyright notices and license terms. -->
     <label name="company"/>
     <field name="company"/>
     <newline/>
+
     <label name="file_"/>
-    <field name="file_"/>
+    <field name="file_" filename_visible="1" colspan="3"/>
+
     <label name="file_format"/>
     <field name="file_format"/>
+    <newline/>
+
+    <label name="csv_format"/>
+    <field name="csv_format" widget="selection"/>
+    <label name="csv_journal"/>
+    <field name="csv_journal" widget="selection"/>
+    <newline/>
 </form>