diff --git a/trytond/CHANGELOG b/trytond/CHANGELOG
index 5971ac6c7b..fc0062413f 100644
--- a/trytond/CHANGELOG
+++ b/trytond/CHANGELOG
@@ -1,3 +1,4 @@
+* Add support for materializing ModelSQL based on a table query
* Add an option to trytond-console to run a script file
* Store only immutable structure in MemoryCache
* Replace ``clean_days`` with ``log_size`` in the cron configuration
diff --git a/trytond/doc/topics/configuration.rst b/trytond/doc/topics/configuration.rst
index cb886eed7f..3d46e8ae67 100644
--- a/trytond/doc/topics/configuration.rst
+++ b/trytond/doc/topics/configuration.rst
@@ -516,6 +516,27 @@ For example::
account.invoice.line = acc_inv_line
account.invoice.tax = acc_inv_tax
+.. _config-table_query_materialized:
+
+table_query_materialized
+------------------------
+
+This section allow to define which :class:`~trytond.model.ModelSQL` with
+:meth:`~trytond.model.ModelSQL.table_query` must be materialized and at which
+interval they must be refreshed.
+
+The interval value is expressed in seconds.
+
+For example::
+
+ [table_query_materialized]
+ account.account.party = 1440
+
+.. note::
+
+ Only the :meth:`~trytond.model.ModelSQL.table_query` that does not depend on
+ the :attr:`~trytond.transaction.Transaction.context`.
+
.. _config-ssl:
ssl
diff --git a/trytond/trytond/backend/database.py b/trytond/trytond/backend/database.py
index efcc649031..9f5e7a3539 100644
--- a/trytond/trytond/backend/database.py
+++ b/trytond/trytond/backend/database.py
@@ -185,6 +185,19 @@ class DatabaseInterface(object):
def has_channel(self):
return False
+ @classmethod
+ def has_materialized_views(cls):
+ return False
+
+ def create_materialized_view(self, connection, view_name, query):
+ raise NotImplementedError
+
+ def drop_materialized_view(self, connection):
+ raise NotImplementedError
+
+ def refresh_materialized_view(self, connection, view_name):
+ raise NotImplementedError
+
def sql_type(self, type_):
pass
diff --git a/trytond/trytond/backend/postgresql/database.py b/trytond/trytond/backend/postgresql/database.py
index 7e4c0d5ec4..a32c5eb490 100644
--- a/trytond/trytond/backend/postgresql/database.py
+++ b/trytond/trytond/backend/postgresql/database.py
@@ -535,6 +535,16 @@ class Database(DatabaseInterface):
if cursor.rowcount:
return schema
+ def get_view_schema(self, connection, view_name):
+ cursor = connection.cursor()
+ for schema in self.search_path:
+ cursor.execute('SELECT 1 '
+ 'FROM pg_matviews '
+ 'WHERE matviewname = %s AND schemaname = %s',
+ (view_name, schema))
+ if cursor.rowcount:
+ return schema
+
@property
def current_user(self):
if self._current_user is None:
@@ -860,6 +870,36 @@ class Database(DatabaseInterface):
def json_contains(self, column, json):
return JSONContains(Cast(column, 'jsonb'), Cast(json, 'jsonb'))
+ @classmethod
+ def has_materialized_views(cls):
+ return True
+
+ def create_materialized_view(self, connection, view_name, query):
+ cursor = connection.cursor()
+ cursor.execute(
+ SQL(
+ "CREATE MATERIALIZED VIEW IF NOT EXISTS {} AS {} "
+ "WITH NO DATA"
+ ).format(Identifier(view_name), SQL(str(query))),
+ query.params)
+ cursor.execute(
+ SQL("CREATE UNIQUE INDEX ON {}(id)").format(Identifier(view_name)))
+
+ def drop_materialized_view(self, connection, view_name):
+ cursor = connection.cursor()
+ cursor.execute(
+ SQL("DROP MATERIALIZED VIEW {}").format(Identifier(view_name)))
+
+ def refresh_materialized_view(
+ self, connection, view_name, concurrently=True):
+ cursor = connection.cursor()
+ cursor.execute(
+ SQL("REFRESH MATERIALIZED VIEW {} {}")
+ .format(
+ SQL('CONCURRENTLY' if concurrently else ''),
+ Identifier(view_name)
+ ))
+
register_type(UNICODE)
if PYDATE:
diff --git a/trytond/trytond/backend/postgresql/table.py b/trytond/trytond/backend/postgresql/table.py
index c334991adf..ce263b43aa 100644
--- a/trytond/trytond/backend/postgresql/table.py
+++ b/trytond/trytond/backend/postgresql/table.py
@@ -32,17 +32,28 @@ class TableHandler(TableHandlerInterface):
cursor = transaction.connection.cursor()
# Create new table if necessary
- if not self.table_exist(self.table_name):
+ if (not (view_exists := self.view_exist(self.table_name))
+ and not self.table_exist(self.table_name)):
cursor.execute(SQL('CREATE TABLE {} ()').format(
Identifier(self.table_name)))
- self.table_schema = transaction.database.get_table_schema(
- transaction.connection, self.table_name)
-
- cursor.execute('SELECT tableowner = current_user FROM pg_tables '
- 'WHERE tablename = %s AND schemaname = %s',
- (self.table_name, self.table_schema))
+ if view_exists:
+ self.table_schema = transaction.database.get_view_schema(
+ transaction.connection, self.table_name)
+ is_owner_query = (
+ 'SELECT matviewowner = current_user FROM pg_matviews '
+ 'WHERE matviewname = %s and schemaname = %s')
+ else:
+ self.table_schema = transaction.database.get_table_schema(
+ transaction.connection, self.table_name)
+ is_owner_query = (
+ 'SELECT tableowner = current_user FROM pg_tables '
+ 'WHERE tablename = %s AND schemaname = %s')
+ cursor.execute(is_owner_query, (self.table_name, self.table_schema))
self.is_owner, = cursor.fetchone()
+ if view_exists:
+ return
+
if model.__doc__ and self.is_owner:
cursor.execute(SQL('COMMENT ON TABLE {} IS %s').format(
Identifier(self.table_name)),
@@ -125,6 +136,12 @@ class TableHandler(TableHandlerInterface):
return bool(transaction.database.get_table_schema(
transaction.connection, table_name))
+ @classmethod
+ def view_exist(cls, view_name):
+ transaction = Transaction()
+ return bool(transaction.database.get_view_schema(
+ transaction.connection, view_name))
+
@classmethod
def table_rename(cls, old_name, new_name):
transaction = Transaction()
diff --git a/trytond/trytond/backend/sqlite/table.py b/trytond/trytond/backend/sqlite/table.py
index d1efb860f4..4d95b7fa90 100644
--- a/trytond/trytond/backend/sqlite/table.py
+++ b/trytond/trytond/backend/sqlite/table.py
@@ -56,6 +56,10 @@ class TableHandler(TableHandlerInterface):
return False
return True
+ @classmethod
+ def view_exist(cls, view_name):
+ return False
+
@classmethod
def table_rename(cls, old_name, new_name):
cursor = Transaction().connection.cursor()
diff --git a/trytond/trytond/backend/table.py b/trytond/trytond/backend/table.py
index 9230fdca89..0948569384 100644
--- a/trytond/trytond/backend/table.py
+++ b/trytond/trytond/backend/table.py
@@ -45,6 +45,10 @@ class TableHandlerInterface(object):
def table_exist(cls, table_name):
raise NotImplementedError
+ @classmethod
+ def view_exist(cls, view_name):
+ raise NotImplementedError
+
@classmethod
def table_rename(cls, old_name, new_name):
raise NotImplementedError
diff --git a/trytond/trytond/ir/action.py b/trytond/trytond/ir/action.py
index 675c57e18a..73e1f53279 100644
--- a/trytond/trytond/ir/action.py
+++ b/trytond/trytond/ir/action.py
@@ -271,6 +271,7 @@ class ActionKeyword(ModelSQL, ModelView):
Action = pool.get('ir.action')
Menu = pool.get('ir.ui.menu')
ModelAccess = pool.get('ir.model.access')
+ ModelData = pool.get('ir.model.data')
User = pool.get('res.user')
groups = User.get_groups()
key = (Transaction().language, groups, keyword, tuple(value))
@@ -279,6 +280,7 @@ class ActionKeyword(ModelSQL, ModelView):
return keywords
keywords = []
model, record_id = value
+ Model = pool.get(model)
clause = [
('keyword', '=', keyword),
@@ -301,6 +303,12 @@ class ActionKeyword(ModelSQL, ModelView):
for action_keyword in action_keywords:
type_ = action_keyword.action.type
types[type_].append(action_keyword.action.id)
+ if (keyword == 'form_action'
+ and issubclass(Model, ModelSQL)
+ and Model._table_query_materialized()):
+ refresh_action = ModelData.get_id(
+ 'ir', 'wizard_model_refresh_materialized')
+ types['ir.action.wizard'].append(refresh_action)
for type_, action_ids in types.items():
for value in Action.get_action_values(type_, action_ids):
if (type_ == 'ir.action.act_window'
diff --git a/trytond/trytond/ir/cron.py b/trytond/trytond/ir/cron.py
index c90a2668da..e0401c759c 100644
--- a/trytond/trytond/ir/cron.py
+++ b/trytond/trytond/ir/cron.py
@@ -94,6 +94,7 @@ class Cron(DeactivableMixin, ModelSQL, ModelView):
('ir.queue|clean', "Clean Task Queue"),
('ir.error|clean', "Clean Errors"),
('ir.cron.log|clean', "Clean Cron Logs"),
+ ('ir.model|refresh_materialized', "Refresh Materialized Models"),
], "Method", required=True, states=_states)
logs = fields.One2Many('ir.cron.log', 'cron', "Logs", readonly=True)
diff --git a/trytond/trytond/ir/model.py b/trytond/trytond/ir/model.py
index c000c982a6..ba1b844092 100644
--- a/trytond/trytond/ir/model.py
+++ b/trytond/trytond/ir/model.py
@@ -27,7 +27,8 @@ from trytond.rpc import RPC
from trytond.tools import cursor_dict, grouped_slice, is_instance_method
from trytond.tools.string_ import StringMatcher
from trytond.transaction import Transaction, without_check_access
-from trytond.wizard import Button, StateAction, StateView, Wizard
+from trytond.wizard import (
+ Button, StateAction, StateTransition, StateView, Wizard)
from .resource import ResourceAccessMixin
@@ -242,6 +243,13 @@ class Model(
def get_name(cls, model):
return cls.get_names().get(model, model)
+ @classmethod
+ def refresh_materialized(cls):
+ pool = Pool()
+ for _, Model in pool.iterobject():
+ if hasattr(Model, '_table_query_refresh'):
+ Model._table_query_refresh()
+
class ModelField(
fields.fmany2one(
@@ -1724,3 +1732,15 @@ class ModelWorkflowGraph(Report):
f'"{record.name}--{to}"',
arrowhead='normal')
subgraph.add_edge(edge)
+
+
+class RefreshMaterialized(Wizard):
+ __name__ = "ir.model.refresh_materialized"
+ start = StateTransition()
+
+ def transition_start(self):
+ pool = Pool()
+ Model = pool.get(Transaction().context['active_model'])
+ if hasattr(Model, '_table_query_refresh'):
+ Model._table_query_refresh(force=True)
+ return 'end'
diff --git a/trytond/trytond/ir/model.xml b/trytond/trytond/ir/model.xml
index ebb496e59d..a33ad86686 100644
--- a/trytond/trytond/ir/model.xml
+++ b/trytond/trytond/ir/model.xml
@@ -329,5 +329,18 @@ this repository contains the full copyright notices and license terms. -->
+
+
+ Refresh Data
+ ir.model.refresh_materialized
+
+
+
+
+
+ ir.model|refresh_materialized
+
+ hours
+
diff --git a/trytond/trytond/ir/tryton.cfg b/trytond/trytond/ir/tryton.cfg
index 93bc14840e..e6a0efa348 100644
--- a/trytond/trytond/ir/tryton.cfg
+++ b/trytond/trytond/ir/tryton.cfg
@@ -121,6 +121,7 @@ wizard:
translation.TranslationReport
ui.view.ShowView
model.PrintModelGraph
+ model.RefreshMaterialized
module.ModuleConfigWizard
module.ModuleActivateUpgrade
module.ModuleConfig
diff --git a/trytond/trytond/model/modelsql.py b/trytond/trytond/model/modelsql.py
index f915ad2b3d..a2777c84e1 100644
--- a/trytond/trytond/model/modelsql.py
+++ b/trytond/trytond/model/modelsql.py
@@ -269,7 +269,7 @@ class Index:
def no_table_query(func):
@wraps(func)
def wrapper(cls, *args, **kwargs):
- if callable(cls.table_query):
+ if cls._is_table_query():
raise NotImplementedError("On table_query")
return func(cls, *args, **kwargs)
return wrapper
@@ -335,11 +335,13 @@ class ModelSQL(ModelStorage):
cls._sql_constraints = []
cls._sql_indexes = set()
cls._history_sql_indexes = set()
- if not callable(cls.table_query):
+ if (not cls._is_table_query()
+ or cls._table_query_materialized()):
table = cls.__table__()
- cls._sql_constraints.append(
- ('id_positive', Check(table, table.id >= 0),
- 'ir.msg_id_positive'))
+ if not cls._is_table_query():
+ cls._sql_constraints.append(
+ ('id_positive', Check(table, table.id >= 0),
+ 'ir.msg_id_positive'))
rec_name_field = getattr(cls, cls._rec_name, None)
if (isinstance(rec_name_field, fields.Field)
and not hasattr(rec_name_field, 'set')):
@@ -428,7 +430,7 @@ class ModelSQL(ModelStorage):
@classmethod
def __table__(cls):
- if callable(cls.table_query):
+ if cls._is_table_query() and not cls._table_query_materialized():
return cls.table_query()
else:
return Table(cls._table)
@@ -445,10 +447,20 @@ class ModelSQL(ModelStorage):
@classmethod
def __register__(cls, module_name):
- cursor = Transaction().connection.cursor()
+ transaction = Transaction()
+ cursor = transaction.connection.cursor()
super().__register__(module_name)
- if callable(cls.table_query):
+ if cls._is_table_query():
+ if transaction.database.has_materialized_views():
+ if backend.TableHandler.view_exist(cls._table):
+ transaction.database.drop_materialized_view(
+ transaction.connection, cls._table)
+ if cls._table_query_materialized():
+ query = cls.table_query()
+ query.columns += (CurrentTimestamp().as_('_refreshed'),)
+ transaction.database.create_materialized_view(
+ transaction.connection, cls._table, query)
return
pool = Pool()
@@ -567,7 +579,8 @@ class ModelSQL(ModelStorage):
if j != index and index < j:
return False
return True
- if not callable(cls.table_query):
+ if (not cls._is_table_query()
+ or cls._table_query_materialized()):
table_h = cls.__table_handler__()
indexes = filter(no_subset, cls._sql_indexes)
table_h.set_indexes(indexes, concurrently=concurrently)
@@ -585,6 +598,36 @@ class ModelSQL(ModelStorage):
continue
history_table.add_column(field_name, field._sql_type)
+ @classmethod
+ def _is_table_query(cls):
+ return callable(cls.table_query)
+
+ @classmethod
+ def _table_query_materialized(cls):
+ if cls._is_table_query():
+ return config.getint(
+ 'table_query_materialized', cls.__name__, default=0)
+
+ @classmethod
+ def _table_query_refresh(cls, concurrently=True, force=False):
+ table = cls.__table__()
+ transaction = Transaction()
+ if not transaction.database.has_materialized_views():
+ return
+ cursor = transaction.connection.cursor()
+ if not (interval := cls._table_query_materialized()):
+ return
+ if not force:
+ cursor.execute(*table.select(
+ CurrentTimestamp() - table._refreshed, limit=1))
+ if row := cursor.fetchone():
+ delta, = row
+ interval = datetime.timedelta(seconds=interval)
+ if delta < interval:
+ return
+ transaction.database.refresh_materialized_view(
+ transaction.connection, cls._table, concurrently=concurrently)
+
@classmethod
@without_check_access
def __raise_integrity_error(
@@ -1126,7 +1169,7 @@ class ModelSQL(ModelStorage):
history_limit = None
if (cls._history
and transaction.context.get('_datetime')
- and not callable(cls.table_query)):
+ and not cls._is_table_query()):
in_max = 1
table = cls.__table_history__()
column = Coalesce(table.write_date, table.create_date)
@@ -1138,7 +1181,7 @@ class ModelSQL(ModelStorage):
for f in all_fields:
field = cls._fields.get(f)
if field and field.sql_type():
- if f in _TABLE_QUERY_COLUMNS and callable(cls.table_query):
+ if f in _TABLE_QUERY_COLUMNS and cls._is_table_query():
column = _TABLE_QUERY_COLUMNS[f]
else:
column = field.sql_column(table)
@@ -1146,7 +1189,7 @@ class ModelSQL(ModelStorage):
if backend.name == 'sqlite':
columns[f].output_name += ' [%s]' % field.sql_type().base
elif f in {'_write', '_delete'}:
- if not callable(cls.table_query):
+ if not cls._is_table_query():
rule_domain = Rule.domain_get(
cls.__name__, mode=f.lstrip('_'))
# No need to compute rule domain if it is the same as the
@@ -1168,7 +1211,7 @@ class ModelSQL(ModelStorage):
columns[f] = rule_expression.as_(f)
else:
columns[f] = Literal(True).as_(f)
- elif f == '_timestamp' and not callable(cls.table_query):
+ elif f == '_timestamp' and not cls._is_table_query():
sql_type = fields.Char('timestamp').sql_type().base
columns[f] = Extract(
'EPOCH', Coalesce(table.write_date, table.create_date)
@@ -1641,7 +1684,7 @@ class ModelSQL(ModelStorage):
if (mode == 'read'
and cls._history
and transaction.context.get('_datetime')
- and not callable(cls.table_query)):
+ and not cls._is_table_query()):
in_max = 1
table = cls.__table_history__()
column = Coalesce(table.write_date, table.create_date)
@@ -1824,7 +1867,7 @@ class ModelSQL(ModelStorage):
columns.append(Column(table, '__id').as_('__id'))
if eager:
- table_query = callable(cls.table_query)
+ table_query = cls._is_table_query()
columns += [f.sql_column(table).as_(n)
for n, f in sorted(cls._fields.items())
if not hasattr(f, 'get')
diff --git a/trytond/trytond/modules/__init__.py b/trytond/trytond/modules/__init__.py
index 60306cd38b..88aaba48b3 100644
--- a/trytond/trytond/modules/__init__.py
+++ b/trytond/trytond/modules/__init__.py
@@ -181,6 +181,7 @@ def load_module_graph(graph, pool, update=None, lang=None, indexes=None):
modules_todo = []
models_to_update_history = set()
models_with_indexes = set()
+ models_to_refresh = set()
# Load also parent languages
lang = set(lang)
@@ -257,6 +258,9 @@ def load_module_graph(graph, pool, update=None, lang=None, indexes=None):
for model in classes['model']:
if hasattr(model, '_history'):
models_to_update_history.add(model.__name__)
+ if (hasattr(model, '_table_query_materialized')
+ and model._table_query_materialized()):
+ models_to_refresh.add(model.__name__)
if hasattr(model, '_update_sql_indexes'):
models_with_indexes.add(model.__name__)
@@ -327,6 +331,10 @@ def load_module_graph(graph, pool, update=None, lang=None, indexes=None):
if update:
cursor.execute(*ir_configuration.update(
[ir_configuration.series], [__series__]))
+ for model_name in models_to_refresh:
+ model = pool.get(model_name)
+ logger.info('refresh %s', model.__name__)
+ model._table_query_refresh(concurrently=False, force=True)
if indexes or indexes is None:
create_indexes(concurrently=False)
else:
diff --git a/trytond/trytond/tests/modelsql.py b/trytond/trytond/tests/modelsql.py
index 956dfc2bd4..0d481e8f71 100644
--- a/trytond/trytond/tests/modelsql.py
+++ b/trytond/trytond/tests/modelsql.py
@@ -1,6 +1,7 @@
# This file is part of Tryton. The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
from sql import Literal
+from sql.aggregate import Count, Min
from sql.operators import Equal
from trytond.model import (
@@ -267,6 +268,36 @@ class ModelLock(ModelSQL):
__name__ = 'test.modelsql.lock'
+class ModelMaterializedViewData(ModelSQL):
+ "Model to provide test data to the materialized views"
+ __name__ = 'test.modelsql.materialized.data'
+ name = fields.Char("Name")
+
+
+class ModelMaterializedView(ModelSQL):
+ "Model to test the materialized views"
+ __name__ = 'test.modelsql.materialized'
+
+ name = fields.Char("Name")
+ count = fields.Integer("Count")
+
+ @classmethod
+ def table_query(cls):
+ pool = Pool()
+ Model = pool.get('test.modelsql.materialized.data')
+ model = Model.__table__()
+
+ return model.select(
+ Min(model.id).as_('id'),
+ model.name.as_('name'),
+ Count(model.id).as_('count'),
+ group_by=[model.name])
+
+ @classmethod
+ def _table_query_materialized(cls):
+ return 1
+
+
def register(module):
Pool.register(
ModelSQLRead,
@@ -297,4 +328,6 @@ def register(module):
ModelUnique,
ModelExclude,
ModelLock,
+ ModelMaterializedViewData,
+ ModelMaterializedView,
module=module, type_='model')
diff --git a/trytond/trytond/tests/test_modelsql.py b/trytond/trytond/tests/test_modelsql.py
index 7b50625eaf..3b9712a2fd 100644
--- a/trytond/trytond/tests/test_modelsql.py
+++ b/trytond/trytond/tests/test_modelsql.py
@@ -1406,6 +1406,31 @@ class ModelSQLTestCase(TestCase):
['OR', ('c', '=', 3), ('d.e', '=', 4)]]
])
+ @unittest.skipUnless(
+ backend.Database.has_materialized_views(),
+ "Database backend does not support materialized views")
+ @with_transaction()
+ def test_table_query_refresh(self):
+ "Test refresh materialized table query"
+ pool = Pool()
+ Data = pool.get('test.modelsql.materialized.data')
+ MaterializedView = pool.get('test.modelsql.materialized')
+
+ Data.create([{
+ 'name': "Foo",
+ }, {
+ 'name': "Foo",
+ }, {
+ 'name': "Bar",
+ }])
+ self.assertEqual(
+ MaterializedView.search([]),
+ [])
+
+ MaterializedView._table_query_refresh()
+ stats = [(s.name, s.count) for s in MaterializedView.search([])]
+ self.assertEqual(stats, [("Foo", 2), ("Bar", 1)])
+
class TranslationTestCase(TestCase):
default_language = 'fr'