tryton/trytond/modules/__init__.py

# This file is part of Tryton.  The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
import os
import sys
import importlib
import itertools
import logging
import configparser
from glob import iglob
from collections import defaultdict
from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES

from sql import Table
from sql.functions import CurrentTimestamp

import trytond.tools as tools
from trytond.config import config
from trytond.exceptions import MissingDependenciesException
from trytond.transaction import Transaction
from trytond import backend
import trytond.convert as convert

logger = logging.getLogger(__name__)

ir_module = Table('ir_module')
ir_model_data = Table('ir_model_data')

OPJ = os.path.join
MODULES_PATH = os.path.abspath(os.path.dirname(__file__))

MODULES = []

EGG_MODULES = {}


def update_egg_modules():
    global EGG_MODULES
    try:
        import pkg_resources
        for ep in pkg_resources.iter_entry_points('trytond.modules'):
            EGG_MODULES[ep.name] = ep
    except ImportError:
        pass


update_egg_modules()


def import_module(name, fullname=None):
    if fullname is None:
        fullname = 'trytond.modules.' + name
    try:
        module = importlib.import_module(fullname)
    except ImportError:
        if name not in EGG_MODULES:
            raise
        ep = EGG_MODULES[name]
        # Can not use ep.load because modules are declared in an importable
        # path and it can not import submodule.
        path = os.path.join(
            ep.dist.location, *ep.module_name.split('.')[:-1])
        if not os.path.isdir(path):
            # Find module in path
            for path in sys.path:
                path = os.path.join(
                    path, *ep.module_name.split('.')[:-1])
                if os.path.isdir(os.path.join(path, name)):
                    break
            else:
                # When testing modules from setuptools location is the
                # module directory
                path = os.path.dirname(ep.dist.location)
        spec = FileFinder(
            path, (SourceFileLoader, SOURCE_SUFFIXES)
            ).find_spec(fullname)
        if spec.loader:
            module = spec.loader.load_module()
        else:
            raise
    return module



def get_module_info(name):
    "Return the content of the tryton.cfg"
    module_config = configparser.ConfigParser()
    with tools.file_open(os.path.join(name, 'tryton.cfg')) as fp:
        module_config.read_file(fp)
        directory = os.path.dirname(fp.name)
    info = dict(module_config.items('tryton'))
    info['directory'] = directory
    for key in ('depends', 'extras_depend', 'xml'):
        if key in info:
            info[key] = info[key].strip().splitlines()
    return info


class Graph(dict):
    def get(self, name):
        if name in self:
            node = self[name]
        else:
            node = self[name] = Node(name)
        return node

    def add(self, name, deps):
        node = self.get(name)
        for dep in deps:
            self.get(dep).append(node)
        return node

    def __iter__(self):
        for node in sorted(self.values(), key=lambda n: (n.depth, n.name)):
            yield node


class Node(list):
    def __init__(self, name):
        super(Node, self).__init__()
        self.name = name
        self.info = None
        self.__depth = 0

    def __repr__(self):
        return str((self.name, self.depth, tuple(self)))

    @property
    def depth(self):
        return self.__depth

    @depth.setter
    def depth(self, value):
        if value > self.__depth:
            self.__depth = value
            for child in self:
                child.depth = value + 1

    def append(self, node):
        assert isinstance(node, Node)
        node.depth = self.depth + 1
        super(Node, self).append(node)


def create_graph(module_list):
    module_list = set(module_list)
    all_deps = set()
    graph = Graph()
    for module in module_list:
        info = get_module_info(module)
        deps = info.get('depends', []) + [
            d for d in info.get('extras_depend', []) if d in module_list]
        node = graph.add(module, deps)
        assert node.info is None
        node.info = info
        all_deps.update(deps)

    missing = all_deps - module_list
    if missing:
        raise MissingDependenciesException(list(missing))
    return graph


def is_module_to_install(module, update):
    if module in update:
        return True
    return False


def load_module_graph(graph, pool, update=None, lang=None):
    from trytond.ir.lang import get_parent_language

    if lang is None:
        lang = [config.get('database', 'language')]
    if update is None:
        update = []
    modules_todo = []
    models_to_update_history = set()

    # Load also parent languages
    lang = set(lang)
    for code in list(lang):
        while code:
            lang.add(code)
            code = get_parent_language(code)

    transaction = Transaction()
    with transaction.connection.cursor() as cursor:
        modules = [x.name for x in graph]
        module2state = dict()
        for sub_modules in tools.grouped_slice(modules):
            cursor.execute(*ir_module.select(ir_module.name, ir_module.state,
                    where=ir_module.name.in_(list(sub_modules))))
            module2state.update(cursor.fetchall())
        modules = set(modules)

        for node in graph:
            module = node.name
            if module not in MODULES:
                continue
            logger.info(module)
            classes = pool.fill(module, modules)
            if update:
                pool.setup(classes)
            package_state = module2state.get(module, 'not activated')
            if (is_module_to_install(module, update)
                    or (update
                        and package_state in ('to activate', 'to upgrade'))):
                if package_state not in ('to activate', 'to upgrade'):
                    if package_state == 'activated':
                        package_state = 'to upgrade'
                    elif package_state != 'to remove':
                        package_state = 'to activate'
                for child in node:
                    module2state[child.name] = package_state
                for type in list(classes.keys()):
                    for cls in classes[type]:
                        logger.info('%s:register %s', module, cls.__name__)
                        cls.__register__(module)
                for model in classes['model']:
                    if hasattr(model, '_history'):
                        models_to_update_history.add(model.__name__)

                # Instanciate a new parser for the module
                tryton_parser = convert.TrytondXmlHandler(
                    pool, module, package_state, modules)

                for filename in node.info.get('xml', []):
                    filename = filename.replace('/', os.sep)
                    logger.info('%s:loading %s', module, filename)
                    # Feed the parser with xml content:
                    with tools.file_open(OPJ(module, filename), 'rb') as fp:
                        tryton_parser.parse_xmlstream(fp)

                modules_todo.append((module, list(tryton_parser.to_delete)))

                localedir = '%s/%s' % (node.info['directory'], 'locale')
                lang2filenames = defaultdict(list)
                for filename in itertools.chain(
                        iglob('%s/*.po' % localedir),
                        iglob('%s/override/*.po' % localedir)):
                    filename = filename.replace('/', os.sep)
                    lang2 = os.path.splitext(os.path.basename(filename))[0]
                    if lang2 not in lang:
                        continue
                    lang2filenames[lang2].append(filename)
                base_path_position = len(node.info['directory']) + 1
                for language, files in lang2filenames.items():
                    filenames = [f[base_path_position:] for f in files]
                    logger.info('%s:loading %s', module, ','.join(filenames))
                    Translation = pool.get('ir.translation')
                    Translation.translation_import(language, module, files)

                if package_state == 'to remove':
                    continue
                cursor.execute(*ir_module.select(ir_module.id,
                        where=(ir_module.name == module)))
                try:
                    module_id, = cursor.fetchone()
                    cursor.execute(*ir_module.update([ir_module.state],
                            ['activated'], where=(ir_module.id == module_id)))
                except TypeError:
                    cursor.execute(*ir_module.insert(
                            [ir_module.create_uid, ir_module.create_date,
                                ir_module.name, ir_module.state],
                            [[0, CurrentTimestamp(), module, 'activated'],
                                ]))
                module2state[module] = 'activated'

            transaction.commit()

        if not update:
            pool.setup()
        else:
            # Remove unknown models and fields
            Model = pool.get('ir.model')
            Model.clean()
            ModelField = pool.get('ir.model.field')
            ModelField.clean()
            transaction.commit()

        pool.setup_mixin(modules)

        for model_name in models_to_update_history:
            model = pool.get(model_name)
            if model._history:
                logger.info('history:update %s', model.__name__)
                model._update_history_table()

        # Vacuum :
        while modules_todo:
            (module, to_delete) = modules_todo.pop()
            convert.post_import(pool, module, to_delete)
    logger.info('all modules loaded')


def get_module_list():
    module_list = set()
    if os.path.exists(MODULES_PATH) and os.path.isdir(MODULES_PATH):
        for file in os.listdir(MODULES_PATH):
            if file.startswith('.'):
                continue
            if file == '__pycache__':
                continue
            if os.path.isdir(OPJ(MODULES_PATH, file)):
                module_list.add(file)
    update_egg_modules()
    module_list.update(EGG_MODULES.keys())
    module_list.add('ir')
    module_list.add('res')
    module_list.add('tests')
    return list(module_list)


def register_classes():
    '''
    Import modules to register the classes in the Pool
    '''
    import trytond.ir
    trytond.ir.register()
    import trytond.res
    trytond.res.register()
    import trytond.tests
    trytond.tests.register()

    for node in create_graph(get_module_list()):
        module = node.name
        logger.info('%s:registering classes', module)

        if module in ('ir', 'res', 'tests'):
            MODULES.append(module)
            continue

        the_module = import_module(module)
        # Some modules register nothing in the Pool
        if hasattr(the_module, 'register'):
            the_module.register()
        MODULES.append(module)


def load_modules(
        database_name, pool, update=None, lang=None, activatedeps=False):
    res = True
    if update:
        update = update[:]
    else:
        update = []

    def _load_modules(update):
        global res
        TableHandler = backend.get('TableHandler')
        transaction = Transaction()

        with transaction.connection.cursor() as cursor:
            # Migration from 3.6: remove double module
            old_table = 'ir_module_module'
            new_table = 'ir_module'
            if TableHandler.table_exist(old_table):
                TableHandler.table_rename(old_table, new_table)

            # Migration from 4.0: rename installed to activated
            cursor.execute(*ir_module.select(ir_module.name,
                    where=ir_module.state.in_(('installed', 'uninstalled'))))
            if cursor.fetchone():
                cursor.execute(*ir_module.update(
                        [ir_module.state], ['activated'],
                        where=ir_module.state == 'installed'))
                cursor.execute(*ir_module.update(
                        [ir_module.state], ['not activated'],
                        where=ir_module.state == 'uninstalled'))

            if update:
                cursor.execute(*ir_module.select(ir_module.name,
                        where=ir_module.state.in_(('activated', 'to activate',
                                'to upgrade', 'to remove'))))
            else:
                cursor.execute(*ir_module.select(ir_module.name,
                        where=ir_module.state.in_(('activated', 'to upgrade',
                                'to remove'))))
            module_list = [name for (name,) in cursor.fetchall()]
            graph = None
            while graph is None:
                module_list += update
                try:
                    graph = create_graph(module_list)
                except MissingDependenciesException as e:
                    if not activatedeps:
                        raise
                    update += e.missings

            load_module_graph(graph, pool, update, lang)

            if update:
                cursor.execute(*ir_module.select(ir_module.name,
                        where=(ir_module.state == 'to remove')))
                fetchall = cursor.fetchall()
                if fetchall:
                    for (mod_name,) in fetchall:
                        # TODO check if ressource not updated by the user
                        cursor.execute(*ir_model_data.select(
                                ir_model_data.model, ir_model_data.db_id,
                                where=(ir_model_data.module == mod_name),
                                order_by=ir_model_data.id.desc))
                        for rmod, rid in cursor.fetchall():
                            Model = pool.get(rmod)
                            Model.delete([Model(rid)])
                        Transaction().connection.commit()
                    cursor.execute(*ir_module.update([ir_module.state],
                            ['not activated'],
                            where=(ir_module.state == 'to remove')))
                    Transaction().connection.commit()
                    res = False

                Module = pool.get('ir.module')
                Module.update_list()
        # Need to commit to unlock SQLite database
        transaction.commit()

    if not Transaction().connection:
        with Transaction().start(database_name, 0):
            _load_modules(update)
    else:
        with Transaction().new_transaction(), \
                Transaction().set_user(0), \
                Transaction().reset_context():
            _load_modules(update)

    return res