diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 235350c5c2..de3fc75b5e 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -5614,7 +5614,8 @@ def archive_deleted_rows_for_table(context, tablename, max_rows): table.c.deleted != default_deleted_value).\ order_by(column).limit(max_rows) - insert_statement = db_utils.InsertFromSelect(shadow_table, query_insert) + insert_statement = sqlalchemyutils.InsertFromSelect( + shadow_table, query_insert) delete_statement = db_utils.DeleteFromSelect(table, query_delete, column) try: # Group the insert and delete in a transaction. diff --git a/nova/db/sqlalchemy/migrate_repo/versions/230_add_details_column_to_instance_actions_events.py b/nova/db/sqlalchemy/migrate_repo/versions/230_add_details_column_to_instance_actions_events.py index c437481a22..0ea1ffa640 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/230_add_details_column_to_instance_actions_events.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/230_add_details_column_to_instance_actions_events.py @@ -15,7 +15,7 @@ from sqlalchemy import Column, String, Text from nova.db.sqlalchemy import api -from nova.db.sqlalchemy import utils +from nova.openstack.common.db.sqlalchemy import utils def upgrade(migrate_engine): diff --git a/nova/db/sqlalchemy/utils.py b/nova/db/sqlalchemy/utils.py index 0b0f2c9f95..79cecc171b 100644 --- a/nova/db/sqlalchemy/utils.py +++ b/nova/db/sqlalchemy/utils.py @@ -13,62 +13,24 @@ # License for the specific language governing permissions and limitations # under the License. -import re - -from migrate.changeset import UniqueConstraint, ForeignKeyConstraint -from sqlalchemy import Boolean -from sqlalchemy import CheckConstraint -from sqlalchemy import Column -from sqlalchemy.engine import reflection from sqlalchemy.exc import OperationalError from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.compiler import compiles -from sqlalchemy import func -from sqlalchemy import Index -from sqlalchemy import Integer from sqlalchemy import MetaData -from sqlalchemy import schema -from sqlalchemy.sql.expression import literal_column from sqlalchemy.sql.expression import UpdateBase -from sqlalchemy.sql import select -from sqlalchemy import String from sqlalchemy import Table from sqlalchemy.types import NullType from nova.db.sqlalchemy import api as db from nova import exception +from nova.openstack.common.db.sqlalchemy import utils as oslodbutils from nova.openstack.common.gettextutils import _ from nova.openstack.common import log as logging -from nova.openstack.common import timeutils LOG = logging.getLogger(__name__) -def get_table(engine, name): - """Returns an sqlalchemy table dynamically from db. - - Needed because the models don't work for us in migrations - as models will be far out of sync with the current data. - """ - metadata = MetaData() - metadata.bind = engine - return Table(name, metadata, autoload=True) - - -class InsertFromSelect(UpdateBase): - def __init__(self, table, select): - self.table = table - self.select = select - - -@compiles(InsertFromSelect) -def visit_insert_from_select(element, compiler, **kw): - return "INSERT INTO %s %s" % ( - compiler.process(element.table, asfrom=True), - compiler.process(element.select)) - - class DeleteFromSelect(UpdateBase): def __init__(self, table, select, column): self.table = table @@ -87,173 +49,6 @@ def visit_delete_from_select(element, compiler, **kw): compiler.process(element.select)) -def _get_not_supported_column(col_name_col_instance, column_name): - try: - column = col_name_col_instance[column_name] - except Exception: - msg = _("Please specify column %s in col_name_col_instance " - "param. It is required because column has unsupported " - "type by sqlite).") - raise exception.NovaException(msg % column_name) - - if not isinstance(column, Column): - msg = _("col_name_col_instance param has wrong type of " - "column instance for column %s It should be instance " - "of sqlalchemy.Column.") - raise exception.NovaException(msg % column_name) - return column - - -def _get_unique_constraints_in_sqlite(migrate_engine, table_name): - regexp = "CONSTRAINT (\w+) UNIQUE \(([^\)]+)\)" - - meta = MetaData(bind=migrate_engine) - table = Table(table_name, meta, autoload=True) - - sql_data = migrate_engine.execute( - """ - SELECT sql - FROM - sqlite_master - WHERE - type = 'table' AND - name = :table_name; - """, - table_name=table_name - ).fetchone()[0] - - uniques = set([ - schema.UniqueConstraint( - *[getattr(table.c, c.strip(' "')) - for c in cols.split(",")], name=name - ) - for name, cols in re.findall(regexp, sql_data) - ]) - - return uniques - - -def _drop_unique_constraint_in_sqlite(migrate_engine, table_name, uc_name, - **col_name_col_instance): - insp = reflection.Inspector.from_engine(migrate_engine) - meta = MetaData(bind=migrate_engine) - - table = Table(table_name, meta, autoload=True) - columns = [] - for column in table.columns: - if isinstance(column.type, NullType): - new_column = _get_not_supported_column(col_name_col_instance, - column.name) - columns.append(new_column) - else: - columns.append(column.copy()) - - uniques = _get_unique_constraints_in_sqlite(migrate_engine, table_name) - table.constraints.update(uniques) - - constraints = [constraint for constraint in table.constraints - if not constraint.name == uc_name and - not isinstance(constraint, schema.ForeignKeyConstraint)] - - new_table = Table(table_name + "__tmp__", meta, *(columns + constraints)) - new_table.create() - - indexes = [] - for index in insp.get_indexes(table_name): - column_names = [new_table.c[c] for c in index['column_names']] - indexes.append(Index(index["name"], - *column_names, - unique=index["unique"])) - f_keys = [] - for fk in insp.get_foreign_keys(table_name): - refcolumns = [fk['referred_table'] + '.' + col - for col in fk['referred_columns']] - f_keys.append(ForeignKeyConstraint(fk['constrained_columns'], - refcolumns, table=new_table, name=fk['name'])) - - ins = InsertFromSelect(new_table, table.select()) - migrate_engine.execute(ins) - table.drop() - - [index.create(migrate_engine) for index in indexes] - for fkey in f_keys: - fkey.create() - new_table.rename(table_name) - - -def drop_unique_constraint(migrate_engine, table_name, uc_name, *columns, - **col_name_col_instance): - """This method drops UC from table and works for mysql, postgresql and - sqlite. In mysql and postgresql we are able to use "alter table" - construction. In sqlite is only one way to drop UC: - 1) Create new table with same columns, indexes and constraints - (except one that we want to drop). - 2) Copy data from old table to new. - 3) Drop old table. - 4) Rename new table to the name of old table. - - :param migrate_engine: sqlalchemy engine - :param table_name: name of table that contains uniq constraint. - :param uc_name: name of uniq constraint that will be dropped. - :param columns: columns that are in uniq constraint. - :param col_name_col_instance: contains pair column_name=column_instance. - column_instance is instance of Column. These params - are required only for columns that have unsupported - types by sqlite. For example BigInteger. - """ - if migrate_engine.name == "sqlite": - _drop_unique_constraint_in_sqlite(migrate_engine, table_name, uc_name, - **col_name_col_instance) - else: - meta = MetaData() - meta.bind = migrate_engine - t = Table(table_name, meta, autoload=True) - uc = UniqueConstraint(*columns, table=t, name=uc_name) - uc.drop() - - -def drop_old_duplicate_entries_from_table(migrate_engine, table_name, - use_soft_delete, *uc_column_names): - """This method is used to drop all old rows that have the same values for - columns in uc_columns. - """ - meta = MetaData() - meta.bind = migrate_engine - - table = Table(table_name, meta, autoload=True) - columns_for_group_by = [table.c[name] for name in uc_column_names] - - columns_for_select = [func.max(table.c.id)] - columns_for_select.extend(list(columns_for_group_by)) - - duplicated_rows_select = select(columns_for_select, - group_by=columns_for_group_by, - having=func.count(table.c.id) > 1) - - for row in migrate_engine.execute(duplicated_rows_select): - # NOTE(boris-42): Do not remove row that has the biggest ID. - delete_condition = table.c.id != row[0] - for name in uc_column_names: - delete_condition &= table.c[name] == row[name] - - rows_to_delete_select = select([table.c.id]).where(delete_condition) - for row in migrate_engine.execute(rows_to_delete_select).fetchall(): - LOG.info(_("Deleted duplicated row with id: %(id)s from table: " - "%(table)s") % dict(id=row[0], table=table_name)) - - if use_soft_delete: - delete_statement = table.update().\ - where(delete_condition).\ - values({ - 'deleted': literal_column('id'), - 'updated_at': literal_column('updated_at'), - 'deleted_at': timeutils.utcnow() - }) - else: - delete_statement = table.delete().where(delete_condition) - migrate_engine.execute(delete_statement) - - def check_shadow_table(migrate_engine, table_name): """This method checks that table with ``table_name`` and corresponding shadow table have same columns. @@ -319,8 +114,8 @@ def create_shadow_table(migrate_engine, table_name=None, table=None, columns = [] for column in table.columns: if isinstance(column.type, NullType): - new_column = _get_not_supported_column(col_name_col_instance, - column.name) + new_column = oslodbutils._get_not_supported_column( + col_name_col_instance, column.name) columns.append(new_column) else: columns.append(column.copy()) @@ -338,272 +133,3 @@ def create_shadow_table(migrate_engine, table_name=None, table=None, except Exception: LOG.info(repr(shadow_table)) LOG.exception(_('Exception while creating table.')) - - -def _get_default_deleted_value(table): - if isinstance(table.c.id.type, Integer): - return 0 - if isinstance(table.c.id.type, String): - return "" - raise exception.NovaException(_("Unsupported id columns type")) - - -def _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes): - table = get_table(migrate_engine, table_name) - - insp = reflection.Inspector.from_engine(migrate_engine) - real_indexes = insp.get_indexes(table_name) - existing_index_names = dict([(index['name'], index['column_names']) - for index in real_indexes]) - - # NOTE(boris-42): Restore indexes on `deleted` column - for index in indexes: - if 'deleted' not in index['column_names']: - continue - name = index['name'] - if name in existing_index_names: - column_names = [table.c[c] for c in existing_index_names[name]] - old_index = Index(name, *column_names, unique=index["unique"]) - old_index.drop(migrate_engine) - - column_names = [table.c[c] for c in index['column_names']] - new_index = Index(index["name"], *column_names, unique=index["unique"]) - new_index.create(migrate_engine) - - -def change_deleted_column_type_to_boolean(migrate_engine, table_name, - **col_name_col_instance): - if migrate_engine.name == "sqlite": - return _change_deleted_column_type_to_boolean_sqlite(migrate_engine, - table_name, - **col_name_col_instance) - insp = reflection.Inspector.from_engine(migrate_engine) - indexes = insp.get_indexes(table_name) - - table = get_table(migrate_engine, table_name) - - old_deleted = Column('old_deleted', Boolean, default=False) - old_deleted.create(table, populate_default=False) - - table.update().\ - where(table.c.deleted == table.c.id).\ - values(old_deleted=True).\ - execute() - - table.c.deleted.drop() - table.c.old_deleted.alter(name="deleted") - - _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes) - - -def _change_deleted_column_type_to_boolean_sqlite(migrate_engine, table_name, - **col_name_col_instance): - insp = reflection.Inspector.from_engine(migrate_engine) - table = get_table(migrate_engine, table_name) - - columns = [] - for column in table.columns: - column_copy = None - if column.name != "deleted": - if isinstance(column.type, NullType): - column_copy = _get_not_supported_column(col_name_col_instance, - column.name) - else: - column_copy = column.copy() - else: - column_copy = Column('deleted', Boolean, default=0) - columns.append(column_copy) - - constraints = [constraint.copy() for constraint in table.constraints] - - meta = MetaData(bind=migrate_engine) - new_table = Table(table_name + "__tmp__", meta, - *(columns + constraints)) - new_table.create() - - indexes = [] - for index in insp.get_indexes(table_name): - column_names = [new_table.c[c] for c in index['column_names']] - indexes.append(Index(index["name"], *column_names, - unique=index["unique"])) - - c_select = [] - for c in table.c: - if c.name != "deleted": - c_select.append(c) - else: - c_select.append(table.c.deleted == table.c.id) - - ins = InsertFromSelect(new_table, select(c_select)) - migrate_engine.execute(ins) - - table.drop() - [index.create(migrate_engine) for index in indexes] - - new_table.rename(table_name) - new_table.update().\ - where(new_table.c.deleted == new_table.c.id).\ - values(deleted=True).\ - execute() - - -def change_deleted_column_type_to_id_type(migrate_engine, table_name, - **col_name_col_instance): - if migrate_engine.name == "sqlite": - return _change_deleted_column_type_to_id_type_sqlite(migrate_engine, - table_name, - **col_name_col_instance) - insp = reflection.Inspector.from_engine(migrate_engine) - indexes = insp.get_indexes(table_name) - - table = get_table(migrate_engine, table_name) - - new_deleted = Column('new_deleted', table.c.id.type, - default=_get_default_deleted_value(table)) - new_deleted.create(table, populate_default=True) - - table.update().\ - where(table.c.deleted == True).\ - values(new_deleted=table.c.id).\ - execute() - table.c.deleted.drop() - table.c.new_deleted.alter(name="deleted") - - _restore_indexes_on_deleted_columns(migrate_engine, table_name, indexes) - - -def _change_deleted_column_type_to_id_type_sqlite(migrate_engine, table_name, - **col_name_col_instance): - # NOTE(boris-42): sqlaclhemy-migrate can't drop column with check - # constraints in sqlite DB and our `deleted` column has - # 2 check constraints. So there is only one way to remove - # these constraints: - # 1) Create new table with the same columns, constraints - # and indexes. (except deleted column). - # 2) Copy all data from old to new table. - # 3) Drop old table. - # 4) Rename new table to old table name. - insp = reflection.Inspector.from_engine(migrate_engine) - meta = MetaData(bind=migrate_engine) - table = Table(table_name, meta, autoload=True) - default_deleted_value = _get_default_deleted_value(table) - - columns = [] - for column in table.columns: - column_copy = None - if column.name != "deleted": - if isinstance(column.type, NullType): - column_copy = _get_not_supported_column(col_name_col_instance, - column.name) - else: - column_copy = column.copy() - else: - column_copy = Column('deleted', table.c.id.type, - default=default_deleted_value) - columns.append(column_copy) - - def is_deleted_column_constraint(constraint): - # NOTE(boris-42): There is no other way to check is CheckConstraint - # associated with deleted column. - if not isinstance(constraint, CheckConstraint): - return False - sqltext = str(constraint.sqltext) - # NOTE(I159): when the type of column `deleted` is changed from boolean - # to int, the corresponding CHECK constraint is dropped too. But - # starting from SQLAlchemy version 0.8.3, those CHECK constraints - # aren't dropped anymore. So despite the fact that column deleted is - # of type int now, we still restrict its values to be either 0 or 1. - constraint_markers = ( - "deleted in (0, 1)", - "deleted IN (:deleted_1, :deleted_2)", - "deleted IN (:param_1, :param_2)" - ) - return any(sqltext.endswith(marker) for marker in constraint_markers) - - constraints = [] - for constraint in table.constraints: - if not is_deleted_column_constraint(constraint): - constraints.append(constraint.copy()) - - new_table = Table(table_name + "__tmp__", meta, - *(columns + constraints)) - new_table.create() - - indexes = [] - for index in insp.get_indexes(table_name): - column_names = [new_table.c[c] for c in index['column_names']] - indexes.append(Index(index["name"], *column_names, - unique=index["unique"])) - - ins = InsertFromSelect(new_table, table.select()) - migrate_engine.execute(ins) - - table.drop() - [index.create(migrate_engine) for index in indexes] - - new_table.rename(table_name) - new_table.update().\ - where(new_table.c.deleted == True).\ - values(deleted=new_table.c.id).\ - execute() - - # NOTE(boris-42): Fix value of deleted column: False -> "" or 0. - new_table.update().\ - where(new_table.c.deleted == False).\ - values(deleted=default_deleted_value).\ - execute() - - -def _index_exists(migrate_engine, table_name, index_name): - inspector = reflection.Inspector.from_engine(migrate_engine) - indexes = inspector.get_indexes(table_name) - index_names = [index['name'] for index in indexes] - - return index_name in index_names - - -def _add_index(migrate_engine, table, index_name, idx_columns): - index = Index( - index_name, *[getattr(table.c, col) for col in idx_columns] - ) - index.create() - - -def _drop_index(migrate_engine, table, index_name, idx_columns): - if _index_exists(migrate_engine, table.name, index_name): - index = Index( - index_name, *[getattr(table.c, col) for col in idx_columns] - ) - index.drop() - - -def _change_index_columns(migrate_engine, table, index_name, - new_columns, old_columns): - _drop_index(migrate_engine, table, index_name, old_columns) - _add_index(migrate_engine, table, index_name, new_columns) - - -def modify_indexes(migrate_engine, data, upgrade=True): - if migrate_engine.name == 'sqlite': - return - - meta = MetaData() - meta.bind = migrate_engine - - for table_name, indexes in data.iteritems(): - table = Table(table_name, meta, autoload=True) - - for index_name, old_columns, new_columns in indexes: - if not upgrade: - new_columns, old_columns = old_columns, new_columns - - if migrate_engine.name == 'postgresql': - if upgrade: - _add_index(migrate_engine, table, index_name, new_columns) - else: - _drop_index(migrate_engine, table, index_name, old_columns) - elif migrate_engine.name == 'mysql': - _change_index_columns(migrate_engine, table, index_name, - new_columns, old_columns) - else: - raise ValueError('Unsupported DB %s' % migrate_engine.name) diff --git a/nova/tests/db/test_db_api.py b/nova/tests/db/test_db_api.py index 45f19e5b36..149cf6ac67 100644 --- a/nova/tests/db/test_db_api.py +++ b/nova/tests/db/test_db_api.py @@ -45,6 +45,7 @@ from nova.db.sqlalchemy import models from nova.db.sqlalchemy import utils as db_utils from nova import exception from nova.openstack.common.db import exception as db_exc +from nova.openstack.common.db.sqlalchemy import utils as sqlalchemyutils from nova.openstack.common import jsonutils from nova.openstack.common import timeutils from nova.openstack.common import uuidutils @@ -6320,22 +6321,24 @@ class ArchiveTestCase(test.TestCase): self.context = context.get_admin_context() self.engine = get_engine() self.conn = self.engine.connect() - self.instance_id_mappings = db_utils.get_table(self.engine, - "instance_id_mappings") - self.shadow_instance_id_mappings = db_utils.get_table(self.engine, - "shadow_instance_id_mappings") - self.dns_domains = db_utils.get_table(self.engine, "dns_domains") - self.shadow_dns_domains = db_utils.get_table(self.engine, - "shadow_dns_domains") - self.consoles = db_utils.get_table(self.engine, "consoles") - self.console_pools = db_utils.get_table(self.engine, "console_pools") - self.shadow_consoles = db_utils.get_table(self.engine, - "shadow_consoles") - self.shadow_console_pools = db_utils.get_table(self.engine, - "shadow_console_pools") - self.instances = db_utils.get_table(self.engine, "instances") - self.shadow_instances = db_utils.get_table(self.engine, - "shadow_instances") + self.instance_id_mappings = sqlalchemyutils.get_table( + self.engine, "instance_id_mappings") + self.shadow_instance_id_mappings = sqlalchemyutils.get_table( + self.engine, "shadow_instance_id_mappings") + self.dns_domains = sqlalchemyutils.get_table( + self.engine, "dns_domains") + self.shadow_dns_domains = sqlalchemyutils.get_table( + self.engine, "shadow_dns_domains") + self.consoles = sqlalchemyutils.get_table(self.engine, "consoles") + self.console_pools = sqlalchemyutils.get_table( + self.engine, "console_pools") + self.shadow_consoles = sqlalchemyutils.get_table( + self.engine, "shadow_consoles") + self.shadow_console_pools = sqlalchemyutils.get_table( + self.engine, "shadow_console_pools") + self.instances = sqlalchemyutils.get_table(self.engine, "instances") + self.shadow_instances = sqlalchemyutils.get_table( + self.engine, "shadow_instances") self.uuidstrs = [] for unused in range(6): self.uuidstrs.append(stdlib_uuid.uuid4().hex) @@ -6349,17 +6352,17 @@ class ArchiveTestCase(test.TestCase): super(ArchiveTestCase, self).tearDown() for tablename in self.id_tablenames_to_cleanup: for name in [tablename, "shadow_" + tablename]: - table = db_utils.get_table(self.engine, name) + table = sqlalchemyutils.get_table(self.engine, name) del_statement = table.delete(table.c.id.in_(self.ids)) self.conn.execute(del_statement) for tablename in self.uuid_tablenames_to_cleanup: for name in [tablename, "shadow_" + tablename]: - table = db_utils.get_table(self.engine, name) + table = sqlalchemyutils.get_table(self.engine, name) del_statement = table.delete(table.c.uuid.in_(self.uuidstrs)) self.conn.execute(del_statement) for tablename in self.domain_tablenames_to_cleanup: for name in [tablename, "shadow_" + tablename]: - table = db_utils.get_table(self.engine, name) + table = sqlalchemyutils.get_table(self.engine, name) del_statement = table.delete(table.c.domain.in_(self.uuidstrs)) self.conn.execute(del_statement) @@ -6438,11 +6441,12 @@ class ArchiveTestCase(test.TestCase): def _test_archive_deleted_rows_for_one_uuid_table(self, tablename): """:returns: 0 on success, 1 if no uuid column, 2 if insert failed.""" - main_table = db_utils.get_table(self.engine, tablename) + main_table = sqlalchemyutils.get_table(self.engine, tablename) if not hasattr(main_table.c, "uuid"): # Not a uuid table, so skip it. return 1 - shadow_table = db_utils.get_table(self.engine, "shadow_" + tablename) + shadow_table = sqlalchemyutils.get_table( + self.engine, "shadow_" + tablename) # Add 6 rows to table for uuidstr in self.uuidstrs: ins_stmt = main_table.insert().values(uuid=uuidstr) diff --git a/nova/tests/db/test_migration_utils.py b/nova/tests/db/test_migration_utils.py index 3584c4178f..d3f93710e3 100644 --- a/nova/tests/db/test_migration_utils.py +++ b/nova/tests/db/test_migration_utils.py @@ -14,22 +14,18 @@ # under the License. import uuid -import warnings -from migrate.changeset import UniqueConstraint import sqlalchemy -from sqlalchemy.dialects import mysql -from sqlalchemy import Boolean, Index, Integer, DateTime, String -from sqlalchemy import MetaData, Table, Column, ForeignKey -from sqlalchemy.engine import reflection +from sqlalchemy import Integer, String +from sqlalchemy import MetaData, Table, Column from sqlalchemy.exc import NoSuchTableError -from sqlalchemy.exc import SAWarning from sqlalchemy.sql import select -from sqlalchemy.types import UserDefinedType, NullType +from sqlalchemy.types import UserDefinedType from nova.db.sqlalchemy import api as db from nova.db.sqlalchemy import utils from nova import exception +from nova.openstack.common.db.sqlalchemy import utils as oslodbutils from nova.tests.db import test_migrations @@ -82,256 +78,6 @@ class TestMigrationUtils(test_migrations.BaseMigrationTestCase): test_table.drop() - def test_insert_from_select(self): - insert_table_name = "__test_insert_to_table__" - select_table_name = "__test_select_from_table__" - uuidstrs = [] - for unused in range(10): - uuidstrs.append(uuid.uuid4().hex) - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - conn = engine.connect() - insert_table = Table(insert_table_name, meta, - Column('id', Integer, primary_key=True, - nullable=False, autoincrement=True), - Column('uuid', String(36), nullable=False)) - select_table = Table(select_table_name, meta, - Column('id', Integer, primary_key=True, - nullable=False, autoincrement=True), - Column('uuid', String(36), nullable=False)) - - insert_table.create() - select_table.create() - # Add 10 rows to select_table - for uuidstr in uuidstrs: - ins_stmt = select_table.insert().values(uuid=uuidstr) - conn.execute(ins_stmt) - - # Select 4 rows in one chunk from select_table - column = select_table.c.id - query_insert = select([select_table], - select_table.c.id < 5).order_by(column) - insert_statement = utils.InsertFromSelect(insert_table, - query_insert) - result_insert = conn.execute(insert_statement) - # Verify we insert 4 rows - self.assertEqual(result_insert.rowcount, 4) - - query_all = select([insert_table]).\ - where(insert_table.c.uuid.in_(uuidstrs)) - rows = conn.execute(query_all).fetchall() - # Verify we really have 4 rows in insert_table - self.assertEqual(len(rows), 4) - - insert_table.drop() - select_table.drop() - - def test_utils_drop_unique_constraint(self): - table_name = "test_utils_drop_unique_constraint" - uc_name = 'uniq_foo' - values = [ - {'id': 1, 'a': 3, 'foo': 10}, - {'id': 2, 'a': 2, 'foo': 20}, - {'id': 3, 'a': 1, 'foo': 30} - ] - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - test_table = Table(table_name, meta, - Column('id', Integer, primary_key=True, - nullable=False), - Column('a', Integer), - Column('foo', Integer), - UniqueConstraint('a', name='uniq_a'), - UniqueConstraint('foo', name=uc_name)) - test_table.create() - - engine.execute(test_table.insert(), values) - # NOTE(boris-42): This method is generic UC dropper. - utils.drop_unique_constraint(engine, table_name, uc_name, 'foo') - - s = test_table.select().order_by(test_table.c.id) - rows = engine.execute(s).fetchall() - - for i in xrange(0, len(values)): - v = values[i] - self.assertEqual((v['id'], v['a'], v['foo']), rows[i]) - - # NOTE(boris-42): Update data about Table from DB. - meta = MetaData() - meta.bind = engine - test_table = Table(table_name, meta, autoload=True) - constraints = filter(lambda c: c.name == uc_name, - test_table.constraints) - self.assertEqual(len(constraints), 0) - self.assertEqual(len(test_table.constraints), 1) - - test_table.drop() - - def test_util_drop_unique_constraint_with_not_supported_sqlite_type(self): - if 'sqlite' not in self.engines: - self.skipTest('sqlite is not configured') - engine = self.engines['sqlite'] - meta = MetaData(bind=engine) - - table_name = ("test_util_drop_unique_constraint_with_not_supported" - "_sqlite_type") - uc_name = 'uniq_foo' - values = [ - {'id': 1, 'a': 3, 'foo': 10}, - {'id': 2, 'a': 2, 'foo': 20}, - {'id': 3, 'a': 1, 'foo': 30} - ] - - test_table = Table(table_name, meta, - Column('id', Integer, primary_key=True, - nullable=False), - Column('a', Integer), - Column('foo', CustomType, default=0), - UniqueConstraint('a', name='uniq_a'), - UniqueConstraint('foo', name=uc_name)) - test_table.create() - - engine.execute(test_table.insert(), values) - warnings.simplefilter("ignore", SAWarning) - - # reflection of custom types has been fixed upstream - if SA_VERSION < (0, 9, 0): - # NOTE(boris-42): Missing info about column `foo` that has - # unsupported type CustomType. - self.assertRaises(exception.NovaException, - utils.drop_unique_constraint, - engine, table_name, uc_name, 'foo') - - # NOTE(boris-42): Wrong type of foo instance. it should be - # instance of sqlalchemy.Column. - self.assertRaises(exception.NovaException, - utils.drop_unique_constraint, - engine, table_name, uc_name, 'foo', - foo=Integer()) - - foo = Column('foo', CustomType, default=0) - utils.drop_unique_constraint(engine, table_name, uc_name, 'foo', - foo=foo) - - s = test_table.select().order_by(test_table.c.id) - rows = engine.execute(s).fetchall() - - for i in xrange(0, len(values)): - v = values[i] - self.assertEqual((v['id'], v['a'], v['foo']), rows[i]) - - # NOTE(boris-42): Update data about Table from DB. - meta = MetaData(bind=engine) - test_table = Table(table_name, meta, autoload=True) - constraints = filter(lambda c: c.name == uc_name, - test_table.constraints) - self.assertEqual(len(constraints), 0) - self.assertEqual(len(test_table.constraints), 1) - test_table.drop() - - def _populate_db_for_drop_duplicate_entries(self, engine, meta, - table_name): - values = [ - {'id': 11, 'a': 3, 'b': 10, 'c': 'abcdef'}, - {'id': 12, 'a': 5, 'b': 10, 'c': 'abcdef'}, - {'id': 13, 'a': 6, 'b': 10, 'c': 'abcdef'}, - {'id': 14, 'a': 7, 'b': 10, 'c': 'abcdef'}, - {'id': 21, 'a': 1, 'b': 20, 'c': 'aa'}, - {'id': 31, 'a': 1, 'b': 20, 'c': 'bb'}, - {'id': 41, 'a': 1, 'b': 30, 'c': 'aef'}, - {'id': 42, 'a': 2, 'b': 30, 'c': 'aef'}, - {'id': 43, 'a': 3, 'b': 30, 'c': 'aef'} - ] - - test_table = Table(table_name, meta, - Column('id', Integer, primary_key=True, - nullable=False), - Column('a', Integer), - Column('b', Integer), - Column('c', String(255)), - Column('deleted', Integer, default=0), - Column('deleted_at', DateTime), - Column('updated_at', DateTime)) - - test_table.create() - engine.execute(test_table.insert(), values) - return test_table, values - - def test_drop_old_duplicate_entries_from_table(self): - table_name = "test_drop_old_duplicate_entries_from_table" - - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - test_table, values = self.\ - _populate_db_for_drop_duplicate_entries(engine, meta, - table_name) - - utils.drop_old_duplicate_entries_from_table(engine, table_name, - False, 'b', 'c') - - uniq_values = set() - expected_ids = [] - for value in sorted(values, key=lambda x: x['id'], reverse=True): - uniq_value = (('b', value['b']), ('c', value['c'])) - if uniq_value in uniq_values: - continue - uniq_values.add(uniq_value) - expected_ids.append(value['id']) - - real_ids = [row[0] for row in - engine.execute(select([test_table.c.id])).fetchall()] - - self.assertEqual(len(real_ids), len(expected_ids)) - for id_ in expected_ids: - self.assertIn(id_, real_ids) - test_table.drop() - - def test_drop_old_duplicate_entries_from_table_soft_delete(self): - table_name = "test_drop_old_duplicate_entries_from_table_soft_delete" - - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - table, values = self.\ - _populate_db_for_drop_duplicate_entries(engine, meta, - table_name) - utils.drop_old_duplicate_entries_from_table(engine, table_name, - True, 'b', 'c') - uniq_values = set() - expected_values = [] - soft_deleted_values = [] - - for value in sorted(values, key=lambda x: x['id'], reverse=True): - uniq_value = (('b', value['b']), ('c', value['c'])) - if uniq_value in uniq_values: - soft_deleted_values.append(value) - continue - uniq_values.add(uniq_value) - expected_values.append(value) - - base_select = table.select() - - rows_select = base_select.\ - where(table.c.deleted != table.c.id) - row_ids = [row['id'] for row in - engine.execute(rows_select).fetchall()] - self.assertEqual(len(row_ids), len(expected_values)) - for value in expected_values: - self.assertIn(value['id'], row_ids) - - deleted_rows_select = base_select.\ - where(table.c.deleted == table.c.id) - deleted_rows_ids = [row['id'] for row in - engine.execute(deleted_rows_select).fetchall()] - self.assertEqual(len(deleted_rows_ids), - len(values) - len(row_ids)) - for value in soft_deleted_values: - self.assertIn(value['id'], deleted_rows_ids) - table.drop() - def test_check_shadow_table(self): table_name = 'test_check_shadow_table' for key, engine in self.engines.items(): @@ -458,7 +204,7 @@ class TestMigrationUtils(test_migrations.BaseMigrationTestCase): # reflection of custom types has been fixed upstream if SA_VERSION < (0, 9, 0): - self.assertRaises(exception.NovaException, + self.assertRaises(oslodbutils.ColumnError, utils.create_shadow_table, engine, table_name=table_name) @@ -508,178 +254,3 @@ class TestMigrationUtils(test_migrations.BaseMigrationTestCase): engine, table_name=table_name) table.drop() shadow_table.drop() - - def test_change_deleted_column_type_doesnt_drop_index(self): - table_name = 'test_change_deleted_column_type_doesnt_drop_index' - for key, engine in self.engines.items(): - meta = MetaData(bind=engine) - - indexes = { - 'idx_a_deleted': ['a', 'deleted'], - 'idx_b_deleted': ['b', 'deleted'], - 'idx_a': ['a'] - } - - index_instances = [Index(name, *columns) - for name, columns in indexes.iteritems()] - - table = Table(table_name, meta, - Column('id', Integer, primary_key=True), - Column('a', String(255)), - Column('b', String(255)), - Column('deleted', Boolean), - *index_instances) - table.create() - utils.change_deleted_column_type_to_id_type(engine, table_name) - utils.change_deleted_column_type_to_boolean(engine, table_name) - - insp = reflection.Inspector.from_engine(engine) - real_indexes = insp.get_indexes(table_name) - self.assertEqual(len(real_indexes), 3) - for index in real_indexes: - name = index['name'] - self.assertIn(name, indexes) - self.assertEqual(set(index['column_names']), - set(indexes[name])) - table.drop() - - def test_change_deleted_column_type_to_id_type_integer(self): - table_name = 'test_change_deleted_column_type_to_id_type_integer' - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - table = Table(table_name, meta, - Column('id', Integer, primary_key=True), - Column('deleted', Boolean)) - table.create() - utils.change_deleted_column_type_to_id_type(engine, table_name) - - table = utils.get_table(engine, table_name) - self.assertIsInstance(table.c.deleted.type, Integer) - table.drop() - - def test_change_deleted_column_type_to_id_type_string(self): - table_name = 'test_change_deleted_column_type_to_id_type_string' - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - table = Table(table_name, meta, - Column('id', String(255), primary_key=True), - Column('deleted', Boolean)) - table.create() - utils.change_deleted_column_type_to_id_type(engine, table_name) - - table = utils.get_table(engine, table_name) - self.assertIsInstance(table.c.deleted.type, String) - table.drop() - - def test_change_deleted_column_type_to_id_type_custom(self): - if 'sqlite' in self.engines: - table_name = 'test_change_deleted_column_type_to_id_type_custom' - engine = self.engines['sqlite'] - meta = MetaData() - meta.bind = engine - table = Table(table_name, meta, - Column('id', Integer, primary_key=True), - Column('foo', CustomType), - Column('deleted', Boolean)) - table.create() - - # reflection of custom types has been fixed upstream - if SA_VERSION < (0, 9, 0): - self.assertRaises(exception.NovaException, - utils.change_deleted_column_type_to_id_type, - engine, table_name) - - fooColumn = Column('foo', CustomType()) - utils.change_deleted_column_type_to_id_type(engine, table_name, - foo=fooColumn) - - table = utils.get_table(engine, table_name) - # NOTE(boris-42): There is no way to check has foo type CustomType. - # but sqlalchemy will set it to NullType. This has - # been fixed upstream in recent SA versions - if SA_VERSION < (0, 9, 0): - self.assertIsInstance(table.c.foo.type, NullType) - self.assertIsInstance(table.c.deleted.type, Integer) - table.drop() - - def test_change_deleted_column_type_to_boolean(self): - table_name = 'test_change_deleted_column_type_to_boolean' - for key, engine in self.engines.items(): - meta = MetaData() - meta.bind = engine - table = Table(table_name, meta, - Column('id', Integer, primary_key=True), - Column('deleted', Integer)) - table.create() - - utils.change_deleted_column_type_to_boolean(engine, table_name) - - table = utils.get_table(engine, table_name) - expected_type = Boolean if key != "mysql" else mysql.TINYINT - self.assertIsInstance(table.c.deleted.type, expected_type) - table.drop() - - def test_change_deleted_column_type_to_boolean_type_custom(self): - if 'sqlite' in self.engines: - table_name = \ - 'test_change_deleted_column_type_to_boolean_type_custom' - engine = self.engines['sqlite'] - meta = MetaData() - meta.bind = engine - table = Table(table_name, meta, - Column('id', Integer, primary_key=True), - Column('foo', CustomType), - Column('deleted', Integer)) - table.create() - - # reflection of custom types has been fixed upstream - if SA_VERSION < (0, 9, 0): - self.assertRaises(exception.NovaException, - utils.change_deleted_column_type_to_boolean, - engine, table_name) - - fooColumn = Column('foo', CustomType()) - utils.change_deleted_column_type_to_boolean(engine, table_name, - foo=fooColumn) - - table = utils.get_table(engine, table_name) - # NOTE(boris-42): There is no way to check has foo type CustomType. - # but sqlalchemy will set it to NullType. This has - # been fixed upstream in recent SA versions. - if SA_VERSION < (0, 9, 0): - self.assertIsInstance(table.c.foo.type, NullType) - self.assertIsInstance(table.c.deleted.type, Boolean) - table.drop() - - def test_drop_unique_constraint_in_sqlite_fk_recreate(self): - if 'sqlite' in self.engines: - engine = self.engines['sqlite'] - meta = MetaData() - meta.bind = engine - parent_table_name = ('test_drop_unique_constraint_in_sqlite_fk_' - 'recreate_parent_table') - parent_table = Table(parent_table_name, meta, - Column('id', Integer, primary_key=True), - Column('foo', Integer)) - parent_table.create() - table_name = 'test_drop_unique_constraint_in_sqlite_fk_recreate' - table = Table(table_name, meta, - Column('id', Integer, primary_key=True), - Column('baz', Integer), - Column('bar', Integer, - ForeignKey(parent_table_name + ".id")), - UniqueConstraint('baz', name='constr1')) - table.create() - utils.drop_unique_constraint(engine, table_name, 'constr1', 'baz') - - insp = reflection.Inspector.from_engine(engine) - f_keys = insp.get_foreign_keys(table_name) - self.assertEqual(len(f_keys), 1) - f_key = f_keys[0] - self.assertEqual(f_key['referred_table'], parent_table_name) - self.assertEqual(f_key['referred_columns'], ['id']) - self.assertEqual(f_key['constrained_columns'], ['bar']) - table.drop() - parent_table.drop() diff --git a/nova/tests/db/test_migrations.py b/nova/tests/db/test_migrations.py index 199aa5590c..dea88ae417 100644 --- a/nova/tests/db/test_migrations.py +++ b/nova/tests/db/test_migrations.py @@ -53,6 +53,7 @@ import sqlalchemy.exc import nova.db.sqlalchemy.migrate_repo from nova.db.sqlalchemy import utils as db_utils +from nova.openstack.common.db.sqlalchemy import utils as oslodbutils from nova.openstack.common.gettextutils import _ from nova.openstack.common import log as logging from nova.openstack.common import processutils @@ -64,46 +65,19 @@ import nova.virt.baremetal.db.sqlalchemy.migrate_repo LOG = logging.getLogger(__name__) -def _get_connect_string(backend, user, passwd, database): - """Try to get a connection with a very specific set of values, if we get - these then we'll run the tests, otherwise they are skipped - """ - if backend == "postgres": - backend = "postgresql+psycopg2" - elif backend == "mysql": - backend = "mysql+mysqldb" - else: - raise Exception("Unrecognized backend: '%s'" % backend) - - return ("%s://%s:%s@localhost/%s" % (backend, user, passwd, database)) - - -def _is_backend_avail(backend, user, passwd, database): - try: - connect_uri = _get_connect_string(backend, user, passwd, database) - engine = sqlalchemy.create_engine(connect_uri) - connection = engine.connect() - except Exception: - # intentionally catch all to handle exceptions even if we don't - # have any backend code loaded. - return False - else: - connection.close() - engine.dispose() - return True - - def _have_mysql(user, passwd, database): present = os.environ.get('NOVA_TEST_MYSQL_PRESENT') if present is None: - return _is_backend_avail('mysql', user, passwd, database) + return oslodbutils.is_backend_avail('mysql+mysqldb', database, + user, passwd) return present.lower() in ('', 'true') def _have_postgresql(user, passwd, database): present = os.environ.get('NOVA_TEST_POSTGRESQL_PRESENT') if present is None: - return _is_backend_avail('postgres', user, passwd, database) + return oslodbutils.is_backend_avail('postgresql+psycopg2', database, + user, passwd) return present.lower() in ('', 'true') @@ -157,8 +131,8 @@ class CommonTestsMixIn(object): """Test that we can trigger a mysql connection failure and we fail gracefully to ensure we don't break people without mysql """ - if _is_backend_avail('mysql', "openstack_cifail", self.PASSWD, - self.DATABASE): + if oslodbutils.is_backend_avail('mysql+mysqldb', self.DATABASE, + "openstack_cifail", self.PASSWD): self.fail("Shouldn't have connected") def test_postgresql_opportunistically(self): @@ -168,8 +142,8 @@ class CommonTestsMixIn(object): """Test that we can trigger a postgres connection failure and we fail gracefully to ensure we don't break people without postgres """ - if _is_backend_avail('postgres', "openstack_cifail", self.PASSWD, - self.DATABASE): + if oslodbutils.is_backend_avail('postgresql+psycopg2', self.DATABASE, + "openstack_cifail", self.PASSWD): self.fail("Shouldn't have connected") @@ -403,8 +377,8 @@ class BaseWalkMigrationTestCase(BaseMigrationTestCase): self.skipTest("mysql not available") # add this to the global lists to make reset work with it, it's removed # automatically in tearDown so no need to clean it up here. - connect_string = _get_connect_string("mysql", self.USER, self.PASSWD, - self.DATABASE) + connect_string = oslodbutils.get_connect_string( + "mysql+mysqldb", self.DATABASE, self.USER, self.PASSWD) (user, password, database, host) = \ get_mysql_connection_info(urlparse.urlparse(connect_string)) engine = sqlalchemy.create_engine(connect_string) @@ -442,8 +416,8 @@ class BaseWalkMigrationTestCase(BaseMigrationTestCase): self.skipTest("postgresql not available") # add this to the global lists to make reset work with it, it's removed # automatically in tearDown so no need to clean it up here. - connect_string = _get_connect_string("postgres", self.USER, - self.PASSWD, self.DATABASE) + connect_string = oslodbutils.get_connect_string( + "postgresql+psycopg2", self.DATABASE, self.USER, self.PASSWD) engine = sqlalchemy.create_engine(connect_string) (user, password, database, host) = \ get_pgsql_connection_info(urlparse.urlparse(connect_string)) @@ -579,26 +553,26 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): self.migration_api = temp.versioning_api def assertColumnExists(self, engine, table, column): - t = db_utils.get_table(engine, table) + t = oslodbutils.get_table(engine, table) self.assertIn(column, t.c) def assertColumnNotExists(self, engine, table, column): - t = db_utils.get_table(engine, table) + t = oslodbutils.get_table(engine, table) self.assertNotIn(column, t.c) def assertTableNotExists(self, engine, table): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - db_utils.get_table, engine, table) + oslodbutils.get_table, engine, table) def assertIndexExists(self, engine, table, index): - t = db_utils.get_table(engine, table) + t = oslodbutils.get_table(engine, table) index_names = [idx.name for idx in t.indexes] self.assertIn(index, index_names) def assertIndexMembers(self, engine, table, index, members): self.assertIndexExists(engine, table, index) - t = db_utils.get_table(engine, table) + t = oslodbutils.get_table(engine, table) index_columns = None for idx in t.indexes: if idx.name == index: @@ -608,7 +582,7 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): self.assertEqual(sorted(members), sorted(index_columns)) def _check_227(self, engine, data): - table = db_utils.get_table(engine, 'project_user_quotas') + table = oslodbutils.get_table(engine, 'project_user_quotas') # Insert fake_quotas with the longest resource name. fake_quotas = {'id': 5, @@ -625,7 +599,7 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): def _check_228(self, engine, data): self.assertColumnExists(engine, 'compute_nodes', 'metrics') - compute_nodes = db_utils.get_table(engine, 'compute_nodes') + compute_nodes = oslodbutils.get_table(engine, 'compute_nodes') self.assertIsInstance(compute_nodes.c.metrics.type, sqlalchemy.types.Text) @@ -635,7 +609,7 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): def _check_229(self, engine, data): self.assertColumnExists(engine, 'compute_nodes', 'extra_resources') - compute_nodes = db_utils.get_table(engine, 'compute_nodes') + compute_nodes = oslodbutils.get_table(engine, 'compute_nodes') self.assertIsInstance(compute_nodes.c.extra_resources.type, sqlalchemy.types.Text) @@ -648,7 +622,8 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): self.assertColumnExists(engine, table_name, 'host') self.assertColumnExists(engine, table_name, 'details') - action_events = db_utils.get_table(engine, 'instance_actions_events') + action_events = oslodbutils.get_table(engine, + 'instance_actions_events') self.assertIsInstance(action_events.c.host.type, sqlalchemy.types.String) self.assertIsInstance(action_events.c.details.type, @@ -663,7 +638,7 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): def _check_231(self, engine, data): self.assertColumnExists(engine, 'instances', 'ephemeral_key_uuid') - instances = db_utils.get_table(engine, 'instances') + instances = oslodbutils.get_table(engine, 'instances') self.assertIsInstance(instances.c.ephemeral_key_uuid.type, sqlalchemy.types.String) self.assertTrue(db_utils.check_shadow_table(engine, 'instances')) @@ -682,25 +657,27 @@ class TestNovaMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): def _check_233(self, engine, data): self.assertColumnExists(engine, 'compute_nodes', 'stats') - compute_nodes = db_utils.get_table(engine, 'compute_nodes') + compute_nodes = oslodbutils.get_table(engine, 'compute_nodes') self.assertIsInstance(compute_nodes.c.stats.type, sqlalchemy.types.Text) - self.assertRaises(sqlalchemy.exc.NoSuchTableError, db_utils.get_table, - engine, 'compute_node_stats') + self.assertRaises(sqlalchemy.exc.NoSuchTableError, + oslodbutils.get_table, engine, 'compute_node_stats') def _post_downgrade_233(self, engine): self.assertColumnNotExists(engine, 'compute_nodes', 'stats') # confirm compute_node_stats exists - db_utils.get_table(engine, 'compute_node_stats') + oslodbutils.get_table(engine, 'compute_node_stats') def _check_244(self, engine, data): - volume_usage_cache = db_utils.get_table(engine, 'volume_usage_cache') + volume_usage_cache = oslodbutils.get_table( + engine, 'volume_usage_cache') self.assertEqual(64, volume_usage_cache.c.user_id.type.length) def _post_downgrade_244(self, engine): - volume_usage_cache = db_utils.get_table(engine, 'volume_usage_cache') + volume_usage_cache = oslodbutils.get_table( + engine, 'volume_usage_cache') self.assertEqual(36, volume_usage_cache.c.user_id.type.length) @@ -740,27 +717,27 @@ class TestBaremetalMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): def _pre_upgrade_002(self, engine): data = [{'id': 1, 'key': 'fake-key', 'image_path': '/dev/null', 'pxe_config_path': '/dev/null/', 'root_mb': 0, 'swap_mb': 0}] - table = db_utils.get_table(engine, 'bm_deployments') + table = oslodbutils.get_table(engine, 'bm_deployments') engine.execute(table.insert(), data) return data def _check_002(self, engine, data): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - db_utils.get_table, engine, 'bm_deployments') + oslodbutils.get_table, engine, 'bm_deployments') def _post_downgrade_004(self, engine): - bm_nodes = db_utils.get_table(engine, 'bm_nodes') + bm_nodes = oslodbutils.get_table(engine, 'bm_nodes') self.assertNotIn(u'instance_name', [c.name for c in bm_nodes.columns]) def _check_005(self, engine, data): - bm_nodes = db_utils.get_table(engine, 'bm_nodes') + bm_nodes = oslodbutils.get_table(engine, 'bm_nodes') columns = [c.name for c in bm_nodes.columns] self.assertNotIn(u'prov_vlan_id', columns) self.assertNotIn(u'registration_status', columns) def _pre_upgrade_006(self, engine): - nodes = db_utils.get_table(engine, 'bm_nodes') - ifs = db_utils.get_table(engine, 'bm_interfaces') + nodes = oslodbutils.get_table(engine, 'bm_nodes') + ifs = oslodbutils.get_table(engine, 'bm_interfaces') # node 1 has two different addresses in bm_nodes and bm_interfaces engine.execute(nodes.insert(), [{'id': 1, @@ -779,7 +756,7 @@ class TestBaremetalMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): 'address': 'cc:cc:cc:cc:cc:cc'}]) def _check_006(self, engine, data): - ifs = db_utils.get_table(engine, 'bm_interfaces') + ifs = oslodbutils.get_table(engine, 'bm_interfaces') rows = ifs.select().\ where(ifs.c.bm_node_id == 1).\ execute().\ @@ -793,7 +770,7 @@ class TestBaremetalMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): self.assertEqual(rows[0]['address'], 'cc:cc:cc:cc:cc:cc') def _post_downgrade_006(self, engine): - ifs = db_utils.get_table(engine, 'bm_interfaces') + ifs = oslodbutils.get_table(engine, 'bm_interfaces') rows = ifs.select().where(ifs.c.bm_node_id == 1).execute().fetchall() self.assertEqual(len(rows), 1) self.assertEqual(rows[0]['address'], 'bb:bb:bb:bb:bb:bb') @@ -802,26 +779,26 @@ class TestBaremetalMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): self.assertEqual(len(rows), 0) def _check_007(self, engine, data): - bm_nodes = db_utils.get_table(engine, 'bm_nodes') + bm_nodes = oslodbutils.get_table(engine, 'bm_nodes') columns = [c.name for c in bm_nodes.columns] self.assertNotIn(u'prov_mac_address', columns) def _check_008(self, engine, data): self.assertRaises(sqlalchemy.exc.NoSuchTableError, - db_utils.get_table, engine, 'bm_pxe_ips') + oslodbutils.get_table, engine, 'bm_pxe_ips') def _post_downgrade_008(self, engine): - db_utils.get_table(engine, 'bm_pxe_ips') + oslodbutils.get_table(engine, 'bm_pxe_ips') def _pre_upgrade_010(self, engine): - bm_nodes = db_utils.get_table(engine, 'bm_nodes') + bm_nodes = oslodbutils.get_table(engine, 'bm_nodes') data = [{'id': 10, 'prov_mac_address': 'cc:cc:cc:cc:cc:cc'}] engine.execute(bm_nodes.insert(), data) return data def _check_010(self, engine, data): - bm_nodes = db_utils.get_table(engine, 'bm_nodes') + bm_nodes = oslodbutils.get_table(engine, 'bm_nodes') self.assertIn('preserve_ephemeral', bm_nodes.columns) default = engine.execute( @@ -833,7 +810,7 @@ class TestBaremetalMigrations(BaseWalkMigrationTestCase, CommonTestsMixIn): bm_nodes.delete().where(bm_nodes.c.id == data[0]['id']).execute() def _post_downgrade_010(self, engine): - bm_nodes = db_utils.get_table(engine, 'bm_nodes') + bm_nodes = oslodbutils.get_table(engine, 'bm_nodes') self.assertNotIn('preserve_ephemeral', bm_nodes.columns) diff --git a/nova/tests/db/test_sqlalchemy_utils.py b/nova/tests/db/test_sqlalchemy_utils.py deleted file mode 100644 index 88b8f071a1..0000000000 --- a/nova/tests/db/test_sqlalchemy_utils.py +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock - -from nova.db.sqlalchemy import utils -from nova import test - - -class SqlAlchemyUtilsTestCase(test.NoDBTestCase): - """Test case for sqlaclchemy utils methods.""" - - def test_modify_indexes_checks_index_before_dropping_in_postgresql(self): - data = {"table_name": (('index2', ('old_column'), - ('new_column')),)} - migrate_engine = mock.Mock() - migrate_engine.name = 'postgresql' - - with mock.patch('nova.db.sqlalchemy.utils.reflection.Inspector' - '.from_engine') as inspector: - inspector.return_value.get_indexes.return_value = [ - {'name': "index1"}] - with mock.patch('nova.db.sqlalchemy.utils.Index') as index: - index.return_value = mock.Mock() - utils.modify_indexes(migrate_engine, data, False) - - self.assertFalse(index.called) - self.assertFalse(index.return_value.drop.called) - - def test_modify_indexes_checks_index_before_dropping_in_mysql(self): - data = {"table_name": (('index2', ('old_column'), - ('new_column')),)} - migrate_engine = mock.Mock() - migrate_engine.name = 'mysql' - - with mock.patch('nova.db.sqlalchemy.utils.reflection.Inspector' - '.from_engine') as inspector: - inspector.return_value.get_indexes.return_value = [ - {'name': "index1"}] - with mock.patch('nova.db.sqlalchemy.utils.Index') as index: - with mock.patch('nova.db.sqlalchemy.utils.Table') as Table: - index.return_value = mock.Mock() - utils.modify_indexes(migrate_engine, data, False) - - self.assertFalse(index.return_value.drop.called) - - def test_modify_indexes(self): - data = {"table_name": (('index2', ('old_column'), - ('new_column')),)} - migrate_engine = mock.Mock() - migrate_engine.name = 'mysql' - - with mock.patch('nova.db.sqlalchemy.utils.reflection.Inspector' - '.from_engine') as inspector: - inspector.return_value.get_indexes.return_value = [ - {'name': "index2"}] - with mock.patch('nova.db.sqlalchemy.utils.Index') as index: - with mock.patch('nova.db.sqlalchemy.utils.Table') as Table: - index.return_value = mock.Mock() - utils.modify_indexes(migrate_engine, data, True) - - self.assertTrue(index.return_value.drop.called) - self.assertTrue(index.return_value.create.called)