From 9e57ec6627f259cfe92cfdd3d0e3eca2a1d38f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Krier?= Date: Fri, 25 Mar 2016 11:25:43 +0100 Subject: [PATCH] Improve TableHandler's to update only modified property It adds keywords argument to _update_definition to limit the update as most of the time only one kind of property is modified. issue5416 review22891002 --- trytond/backend/mysql/table.py | 128 +++++++++++++------------- trytond/backend/postgresql/table.py | 133 +++++++++++++++------------- trytond/backend/sqlite/table.py | 76 ++++++++-------- 3 files changed, 174 insertions(+), 163 deletions(-) diff --git a/trytond/backend/mysql/table.py b/trytond/backend/mysql/table.py index 903cd40620..a7bd9d2841 100644 --- a/trytond/backend/mysql/table.py +++ b/trytond/backend/mysql/table.py @@ -17,7 +17,6 @@ class TableHandler(TableHandlerInterface): self._constraints = [] self._fkeys = [] self._indexes = [] - self._field2module = {} self._model = model cursor = Transaction().connection.cursor() @@ -35,7 +34,7 @@ class TableHandler(TableHandlerInterface): 'PRIMARY KEY(__id)' ') ENGINE=InnoDB;' % self.table_name) - self._update_definitions() + self._update_definitions(columns=True) if 'id' not in self._columns: if not self.history: cursor.execute('ALTER TABLE `%s` ' @@ -44,7 +43,7 @@ class TableHandler(TableHandlerInterface): else: cursor.execute('ALTER TABLE `%s` ' 'ADD COLUMN id BIGINT' % self.table_name) - self._update_definitions() + self._update_definitions(columns=True) if self.history and '__id' not in self._columns: cursor.execute('ALTER TABLE `%s` ' 'ADD COLUMN __id BIGINT AUTO_INCREMENT ' @@ -94,61 +93,66 @@ class TableHandler(TableHandlerInterface): cursor.execute('ALTER TABLE `%s` ' 'RENAME COLUMN `%s` TO `%s`' % (self.table_name, old_name, new_name)) - self._update_definitions() + self._update_definitions(columns=True) elif exception and self.column_exist(new_name): raise Exception('Unable to rename column %s.%s to %s.%s: ' '%s.%s already exist!' % (self.table_name, old_name, self.table_name, new_name, self.table_name, new_name)) - def _update_definitions(self): + def _update_definitions(self, + columns=None, constraints=None, indexes=None): + if columns is None and constraints is None and indexes is None: + columns = constraints = indexes = True transaction = Transaction() cursor = transaction.connection.cursor() - # Fetch columns definitions from the table - cursor.execute("SELECT column_name, character_maximum_length, " - "data_type, is_nullable, column_default " - "FROM information_schema.columns " - "WHERE table_schema = %s AND table_name = %s", - (transaction.database.name, self.table_name)) - self._columns = {} - for line in cursor.fetchall(): - column, size, typname, nullable, default = line - self._columns[column] = { - 'size': size, - 'typname': typname, - 'nullable': nullable == 'YES' and True or False, - 'default': default, - } - - # fetch constraints for the table - cursor.execute("SELECT constraint_name, constraint_type " - "FROM information_schema.table_constraints " - "WHERE table_schema = %s AND table_name = %s", - (transaction.database.name, self.table_name)) - self._constraints = [] - self._fkeys = [] - for line in cursor.fetchall(): - conname, contype = line - if contype not in ('PRIMARY KEY', 'FOREIGN KEY'): - self._constraints.append(conname) - elif contype == 'FOREIGN KEY': - self._fkeys.append(conname) - - # Fetch indexes defined for the table - cursor.execute('SHOW INDEXES FROM `%s`' % self.table_name) - self._indexes = list(set(x[2] for x in cursor.fetchall() - if x[2] != 'PRIMARY')) - - # Keep track of which module created each field - self._field2module = {} - if self.object_name is not None: - cursor.execute('SELECT f.name, f.module ' - 'FROM ir_model_field f ' - 'JOIN ir_model m on (f.model=m.id) ' - 'WHERE m.model = %s', - (self.object_name,)) + if columns: + # Fetch columns definitions from the table + cursor.execute("SELECT column_name, character_maximum_length, " + "data_type, is_nullable, column_default " + "FROM information_schema.columns " + "WHERE table_schema = %s AND table_name = %s", + (transaction.database.name, self.table_name)) + self._columns = {} for line in cursor.fetchall(): - self._field2module[line[0]] = line[1] + column, size, typname, nullable, default = line + self._columns[column] = { + 'size': size, + 'typname': typname, + 'nullable': nullable == 'YES' and True or False, + 'default': default, + } + + if constraints: + # fetch constraints for the table + cursor.execute("SELECT constraint_name, constraint_type " + "FROM information_schema.table_constraints " + "WHERE table_schema = %s AND table_name = %s", + (transaction.database.name, self.table_name)) + self._constraints = [] + self._fkeys = [] + for line in cursor.fetchall(): + conname, contype = line + if contype not in ('PRIMARY KEY', 'FOREIGN KEY'): + self._constraints.append(conname) + elif contype == 'FOREIGN KEY': + self._fkeys.append(conname) + + if indexes: + # Fetch indexes defined for the table + cursor.execute('SHOW INDEXES FROM `%s`' % self.table_name) + self._indexes = list(set(x[2] for x in cursor.fetchall() + if x[2] != 'PRIMARY')) + + @property + def _field2module(self): + cursor = Transaction().connection.cursor() + cursor.execute('SELECT f.name, f.module ' + 'FROM ir_model_field f ' + 'JOIN ir_model m on (f.model=m.id) ' + 'WHERE m.model = %s', + (self.object_name,)) + return dict(cursor) def alter_size(self, column_name, column_type): cursor = Transaction().connection.cursor() @@ -156,7 +160,7 @@ class TableHandler(TableHandlerInterface): 'MODIFY COLUMN `%s` %s' % (self.table_name, column_name, self._column_definition(column_name))) - self._update_definitions() + self._update_definitions(columns=True) def alter_type(self, column_name, column_type): cursor = Transaction().connection.cursor() @@ -164,7 +168,7 @@ class TableHandler(TableHandlerInterface): 'MODIFY COLUMN `%s` %s' % (self.table_name, column_name, self._column_definition(column_name, typname=column_type))) - self._update_definitions() + self._update_definitions(columns=True) def db_default(self, column_name, value): cursor = Transaction().connection.cursor() @@ -172,7 +176,7 @@ class TableHandler(TableHandlerInterface): 'MODIFY COLUMN `%s` %s' % (self.table_name, column_name, self._column_definition(column_name, default=value))) - self._update_definitions() + self._update_definitions(columns=True) def add_raw_column(self, column_name, column_type, column_format, default_fun=None, field_size=None, migrate=True, string=''): @@ -237,7 +241,7 @@ class TableHandler(TableHandlerInterface): 'SET `' + column_name + '` = %s', (column_format(default),)) - self._update_definitions() + self._update_definitions(columns=True) def add_fk(self, column_name, reference, on_delete=None): if on_delete is None: @@ -251,7 +255,7 @@ class TableHandler(TableHandlerInterface): 'REFERENCES `%s` (id) ON DELETE %s' % (self.table_name, conname, column_name, reference, on_delete)) - self._update_definitions() + self._update_definitions(constraints=True) def drop_fk(self, column_name, table=None): conname = '%s_%s_fkey' % (self.table_name, column_name) @@ -260,7 +264,7 @@ class TableHandler(TableHandlerInterface): cursor = Transaction().connection.cursor() cursor.execute('ALTER TABLE `%s` ' 'DROP FOREIGN KEY `%s`' % (self.table_name, conname)) - self._update_definitions() + self._update_definitions(constraints=True) def index_action(self, column_name, action='add', table=None): if isinstance(column_name, basestring): @@ -284,7 +288,7 @@ class TableHandler(TableHandlerInterface): + '` ( ' + ','.join(['`' + x + '`' for x in column_name]) + ')') - self._update_definitions() + self._update_definitions(indexes=True) elif action == 'remove': if len(column_name) == 1: if (self._field2module.get(column_name[0], @@ -294,7 +298,7 @@ class TableHandler(TableHandlerInterface): if index_name in self._indexes: cursor.execute('DROP INDEX `%s` ON `%s`' % (index_name, self.table_name)) - self._update_definitions() + self._update_definitions(indexes=True) else: raise Exception('Index action not supported!') @@ -315,7 +319,7 @@ class TableHandler(TableHandlerInterface): % (self.table_name, column_name, self._column_definition(column_name, nullable=False))) - self._update_definitions() + self._update_definitions(columns=True) else: logger.warning( 'Unable to set column %s ' @@ -338,7 +342,7 @@ class TableHandler(TableHandlerInterface): 'MODIFY COLUMN `%s` %s' % (self.table_name, column_name, self._column_definition(column_name, nullable=True))) - self._update_definitions() + self._update_definitions(columns=True) else: raise Exception('Not null action not supported!') @@ -363,7 +367,7 @@ class TableHandler(TableHandlerInterface): 'ALTER table `%s` ADD CONSTRAINT `%s` %s', constraint, self.table_name, self.table_name, ident, constraint, exc_info=True) - self._update_definitions() + self._update_definitions(constraints=True) def drop_constraint(self, ident, exception=False, table=None): ident = (table or self.table_name) + "_" + ident @@ -381,7 +385,7 @@ class TableHandler(TableHandlerInterface): logger.warning( 'unable to drop \'%s\' constraint on table %s!', ident, self.table_name) - self._update_definitions() + self._update_definitions(constraints=True) def drop_column(self, column_name, exception=False): if not self.column_exist(column_name): @@ -399,7 +403,7 @@ class TableHandler(TableHandlerInterface): logger.warning( 'unable to drop \'%s\' column on table %s!', column_name, self.table_name) - self._update_definitions() + self._update_definitions(columns=True) @staticmethod def drop_table(model, table, cascade=False): diff --git a/trytond/backend/postgresql/table.py b/trytond/backend/postgresql/table.py index 9250749f24..8b9fa9362c 100644 --- a/trytond/backend/postgresql/table.py +++ b/trytond/backend/postgresql/table.py @@ -18,7 +18,6 @@ class TableHandler(TableHandlerInterface): self._constraints = [] self._fk_deltypes = {} self._indexes = [] - self._field2module = {} transaction = Transaction() cursor = transaction.connection.cursor() @@ -41,7 +40,7 @@ class TableHandler(TableHandlerInterface): cursor.execute('COMMENT ON TABLE "%s" IS \'%s\'' % (self.table_name, model.__doc__.replace("'", "''"))) - self._update_definitions() + self._update_definitions(columns=True) if 'id' not in self._columns: if not self.history: cursor.execute('ALTER TABLE "%s" ' @@ -53,7 +52,7 @@ class TableHandler(TableHandlerInterface): else: cursor.execute('ALTER TABLE "%s" ' 'ADD COLUMN id INTEGER' % self.table_name) - self._update_definitions() + self._update_definitions(columns=True) if self.history and '__id' not in self._columns: cursor.execute('ALTER TABLE "%s" ' 'ADD COLUMN __id INTEGER ' @@ -134,67 +133,73 @@ class TableHandler(TableHandlerInterface): cursor.execute('ALTER TABLE "%s" ' 'RENAME COLUMN "%s" TO "%s"' % (self.table_name, old_name, new_name)) - self._update_definitions() + self._update_definitions(columns=True) elif exception and self.column_exist(new_name): raise Exception('Unable to rename column %s.%s to %s.%s: ' '%s.%s already exist!' % (self.table_name, old_name, self.table_name, new_name, self.table_name, new_name)) - def _update_definitions(self): + def _update_definitions(self, + columns=None, constraints=None, indexes=None): + if columns is None and constraints is None and indexes is None: + columns = constraints = indexes = True cursor = Transaction().connection.cursor() - self._columns = {} - # Fetch columns definitions from the table - cursor.execute('SELECT ' - 'column_name, udt_name, is_nullable, character_maximum_length, ' - 'column_default ' - 'FROM information_schema.columns ' - 'WHERE table_name = %s AND table_schema = %s', - (self.table_name, self.table_schema)) - for column, typname, nullable, size, default in cursor.fetchall(): - self._columns[column] = { - 'typname': typname, - 'notnull': True if nullable == 'NO' else False, - 'size': size, - 'default': default, - } + if columns: + self._columns = {} + # Fetch columns definitions from the table + cursor.execute('SELECT ' + 'column_name, udt_name, is_nullable, ' + 'character_maximum_length, ' + 'column_default ' + 'FROM information_schema.columns ' + 'WHERE table_name = %s AND table_schema = %s', + (self.table_name, self.table_schema)) + for column, typname, nullable, size, default in cursor.fetchall(): + self._columns[column] = { + 'typname': typname, + 'notnull': True if nullable == 'NO' else False, + 'size': size, + 'default': default, + } - # fetch constraints for the table - cursor.execute('SELECT constraint_name ' - 'FROM information_schema.table_constraints ' - 'WHERE table_name = %s AND table_schema = %s', - (self.table_name, self.table_schema)) - self._constraints = [c for c, in cursor.fetchall()] + if constraints: + # fetch constraints for the table + cursor.execute('SELECT constraint_name ' + 'FROM information_schema.table_constraints ' + 'WHERE table_name = %s AND table_schema = %s', + (self.table_name, self.table_schema)) + self._constraints = [c for c, in cursor.fetchall()] - cursor.execute('SELECT k.column_name, r.delete_rule ' - 'FROM information_schema.key_column_usage AS k ' - 'JOIN information_schema.referential_constraints AS r ' - 'ON r.constraint_schema = k.constraint_schema ' - 'AND r.constraint_name = k.constraint_name ' - 'WHERE k.table_name = %s AND k.table_schema = %s', - (self.table_name, self.table_schema)) - self._fk_deltypes = dict(cursor.fetchall()) + cursor.execute('SELECT k.column_name, r.delete_rule ' + 'FROM information_schema.key_column_usage AS k ' + 'JOIN information_schema.referential_constraints AS r ' + 'ON r.constraint_schema = k.constraint_schema ' + 'AND r.constraint_name = k.constraint_name ' + 'WHERE k.table_name = %s AND k.table_schema = %s', + (self.table_name, self.table_schema)) + self._fk_deltypes = dict(cursor.fetchall()) - # Fetch indexes defined for the table - cursor.execute("SELECT cl2.relname " - "FROM pg_index ind " - "JOIN pg_class cl on (cl.oid = ind.indrelid) " - "JOIN pg_namespace n ON (cl.relnamespace = n.oid) " - "JOIN pg_class cl2 on (cl2.oid = ind.indexrelid) " - "WHERE cl.relname = %s AND n.nspname = %s", - (self.table_name, self.table_schema)) - self._indexes = [l[0] for l in cursor.fetchall()] + if indexes: + # Fetch indexes defined for the table + cursor.execute("SELECT cl2.relname " + "FROM pg_index ind " + "JOIN pg_class cl on (cl.oid = ind.indrelid) " + "JOIN pg_namespace n ON (cl.relnamespace = n.oid) " + "JOIN pg_class cl2 on (cl2.oid = ind.indexrelid) " + "WHERE cl.relname = %s AND n.nspname = %s", + (self.table_name, self.table_schema)) + self._indexes = [l[0] for l in cursor.fetchall()] - # Keep track of which module created each field - self._field2module = {} - if self.object_name is not None: - cursor.execute('SELECT f.name, f.module ' - 'FROM ir_model_field f ' - 'JOIN ir_model m on (f.model=m.id) ' - 'WHERE m.model = %s', - (self.object_name,)) - for line in cursor: - self._field2module[line[0]] = line[1] + @property + def _field2module(self): + cursor = Transaction().connection.cursor() + cursor.execute('SELECT f.name, f.module ' + 'FROM ir_model_field f ' + 'JOIN ir_model m on (f.model=m.id) ' + 'WHERE m.model = %s', + (self.object_name,)) + return dict(cursor) def alter_size(self, column_name, column_type): cursor = Transaction().connection.cursor() @@ -210,13 +215,13 @@ class TableHandler(TableHandlerInterface): cursor.execute("ALTER TABLE \"%s\" " "DROP COLUMN _temp_change_size" % (self.table_name,)) - self._update_definitions() + self._update_definitions(columns=True) def alter_type(self, column_name, column_type): cursor = Transaction().connection.cursor() cursor.execute('ALTER TABLE "' + self.table_name + '" ' 'ALTER "' + column_name + '" TYPE ' + column_type) - self._update_definitions() + self._update_definitions(columns=True) def db_default(self, column_name, value): if value in [True, False]: @@ -300,7 +305,7 @@ class TableHandler(TableHandlerInterface): 'SET "' + column_name + '" = %s', (column_format(default),)) - self._update_definitions() + self._update_definitions(columns=True) def add_fk(self, column_name, reference, on_delete=None): if on_delete is not None: @@ -327,7 +332,7 @@ class TableHandler(TableHandlerInterface): 'FOREIGN KEY ("' + column_name + '") ' 'REFERENCES "' + reference + '" ' 'ON DELETE ' + on_delete) - self._update_definitions() + self._update_definitions(constraints=True) def drop_fk(self, column_name, table=None): self.drop_constraint(column_name + '_fkey', table=table) @@ -349,7 +354,7 @@ class TableHandler(TableHandlerInterface): cursor.execute('CREATE INDEX "' + index_name + '" ' 'ON "' + self.table_name + '" ( ' + ','.join(['"' + x + '"' for x in column_name]) + ')') - self._update_definitions() + self._update_definitions(indexes=True) elif action == 'remove': if len(column_name) == 1: if (self._field2module.get(column_name[0], @@ -358,7 +363,7 @@ class TableHandler(TableHandlerInterface): if test_index_name in self._indexes: cursor.execute('DROP INDEX "%s" ' % (index_name,)) - self._update_definitions() + self._update_definitions(indexes=True) else: raise Exception('Index action not supported!') @@ -376,7 +381,7 @@ class TableHandler(TableHandlerInterface): if not cursor.rowcount: cursor.execute('ALTER TABLE "' + self.table_name + '" ' 'ALTER COLUMN "' + column_name + '" SET NOT NULL') - self._update_definitions() + self._update_definitions(columns=True) else: logger.warning( 'Unable to set column %s ' @@ -397,7 +402,7 @@ class TableHandler(TableHandlerInterface): cursor.execute('ALTER TABLE "%s" ' 'ALTER COLUMN "%s" DROP NOT NULL' % (self.table_name, column_name)) - self._update_definitions() + self._update_definitions(columns=True) else: raise Exception('Not null action not supported!') @@ -421,7 +426,7 @@ class TableHandler(TableHandlerInterface): 'ALTER table "%s" ADD CONSTRAINT "%s" %s', constraint, self.table_name, self.table_name, ident, constraint) - self._update_definitions() + self._update_definitions(constraints=True) def drop_constraint(self, ident, exception=False, table=None): ident = (table or self.table_name) + "_" + ident @@ -438,7 +443,7 @@ class TableHandler(TableHandlerInterface): logger.warning( 'unable to drop \'%s\' constraint on table %s!', ident, self.table_name) - self._update_definitions() + self._update_definitions(constraints=True) def drop_column(self, column_name, exception=False): if not self.column_exist(column_name): @@ -455,7 +460,7 @@ class TableHandler(TableHandlerInterface): logger.warning( 'unable to drop \'%s\' column on table %s!', column_name, self.table_name, exc_info=True) - self._update_definitions() + self._update_definitions(columns=True) @staticmethod def drop_table(cursor, model, table, cascade=False): diff --git a/trytond/backend/sqlite/table.py b/trytond/backend/sqlite/table.py index 56fb9f26c7..1609edebc9 100644 --- a/trytond/backend/sqlite/table.py +++ b/trytond/backend/sqlite/table.py @@ -20,7 +20,6 @@ class TableHandler(TableHandlerInterface): self._constraints = [] self._fk_deltypes = {} self._indexes = [] - self._field2module = {} self._model = model cursor = Transaction().connection.cursor() @@ -126,44 +125,47 @@ class TableHandler(TableHandlerInterface): % (self.table_name, old_name, self.table_name, new_name, self.table_name, new_name)) - def _update_definitions(self): + def _update_definitions(self, columns=None, indexes=None): + if columns is None and indexes is None: + columns = indexes = True cursor = Transaction().connection.cursor() # Fetch columns definitions from the table - cursor.execute('PRAGMA table_info("' + self.table_name + '")') - self._columns = {} - for _, column, type_, notnull, hasdef, _ in cursor.fetchall(): - column = re.sub(r'^\"|\"$', '', column) - match = re.match(r'(\w+)(\((.*?)\))?', type_) - if match: - typname = match.group(1).upper() - size = match.group(3) and int(match.group(3)) or 0 - else: - typname = type_.upper() - size = -1 - self._columns[column] = { - 'notnull': notnull, - 'hasdef': hasdef, - 'size': size, - 'typname': typname, - } + if columns: + cursor.execute('PRAGMA table_info("' + self.table_name + '")') + self._columns = {} + for _, column, type_, notnull, hasdef, _ in cursor.fetchall(): + column = re.sub(r'^\"|\"$', '', column) + match = re.match(r'(\w+)(\((.*?)\))?', type_) + if match: + typname = match.group(1).upper() + size = match.group(3) and int(match.group(3)) or 0 + else: + typname = type_.upper() + size = -1 + self._columns[column] = { + 'notnull': notnull, + 'hasdef': hasdef, + 'size': size, + 'typname': typname, + } # Fetch indexes defined for the table - try: - cursor.execute('PRAGMA index_list("' + self.table_name + '")') - except IndexError: # There is sometimes IndexError - cursor.execute('PRAGMA index_list("' + self.table_name + '")') - self._indexes = [l[1] for l in cursor.fetchall()] + if indexes: + try: + cursor.execute('PRAGMA index_list("' + self.table_name + '")') + except IndexError: # There is sometimes IndexError + cursor.execute('PRAGMA index_list("' + self.table_name + '")') + self._indexes = [l[1] for l in cursor.fetchall()] - # Keep track of which module created each field - self._field2module = {} - if self.object_name is not None: - cursor.execute('SELECT f.name, f.module ' - 'FROM ir_model_field f ' - 'JOIN ir_model m on (f.model=m.id) ' - 'WHERE m.model = ?', - (self.object_name,)) - for line in cursor.fetchall(): - self._field2module[line[0]] = line[1] + @property + def _field2module(self): + cursor = Transaction().connection.cursor() + cursor.execute('SELECT f.name, f.module ' + 'FROM ir_model_field f ' + 'JOIN ir_model m on (f.model=m.id) ' + 'WHERE m.model = ?', + (self.object_name,)) + return dict(cursor) def alter_size(self, column_name, column_type): warnings.warn('Unable to alter size of column with SQLite backend') @@ -234,7 +236,7 @@ class TableHandler(TableHandlerInterface): 'SET "' + column_name + '" = ?', (column_format(default),)) - self._update_definitions() + self._update_definitions(columns=True) def add_fk(self, column_name, reference, on_delete=None): warnings.warn('Unable to add foreign key with SQLite backend') @@ -255,7 +257,7 @@ class TableHandler(TableHandlerInterface): 'ON "' + self.table_name + '" ( ' + ','.join('"' + x + '"' for x in column_name) + ')') - self._update_definitions() + self._update_definitions(indexes=True) elif action == 'remove': if len(column_name) == 1: if self._field2module.get(column_name[0], @@ -264,7 +266,7 @@ class TableHandler(TableHandlerInterface): if index_name in self._indexes: cursor.execute('DROP INDEX "%s" ' % (index_name,)) - self._update_definitions() + self._update_definitions(indexes=True) else: raise Exception('Index action not supported!')