From 55f59172ee708b0f927b08146bc76219715b3662 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 1 Dec 2017 17:13:07 +0800 Subject: [PATCH] Add cross cell sort support for get_migrations These will be used by migrations cross cell list. We can get migrations objects with limit/marker/sort params from multiple cell using get_migrations_sorted. Change-Id: I2a6d21752b385ebd1e95ec9fe3d18b78a4017bb3 --- nova/compute/api.py | 8 ++ nova/compute/migration_list.py | 86 ++++++++++++++++ nova/db/api.py | 11 +++ nova/db/sqlalchemy/api.py | 26 +++++ .../functional/compute/test_migration_list.py | 98 +++++++++++++++++++ nova/tests/unit/db/test_db_api.py | 8 ++ 6 files changed, 237 insertions(+) create mode 100644 nova/compute/migration_list.py create mode 100644 nova/tests/functional/compute/test_migration_list.py diff --git a/nova/compute/api.py b/nova/compute/api.py index 199fe6d9ea..a5ee2fad26 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -44,6 +44,7 @@ from nova.cells import opts as cells_opts from nova.compute import flavors from nova.compute import instance_actions from nova.compute import instance_list +from nova.compute import migration_list from nova.compute import power_state from nova.compute import rpcapi as compute_rpcapi from nova.compute import task_states @@ -4250,6 +4251,13 @@ class API(base.Base): cctxt, filters).objects) return objects.MigrationList(objects=migrations) + def get_migrations_sorted(self, context, filters, sort_dirs=None, + sort_keys=None, limit=None, marker=None): + """Get all migrations for the given parameters.""" + mig_objs = migration_list.get_migration_objects_sorted( + context, filters, limit, marker, sort_keys, sort_dirs) + return mig_objs + def get_migrations_in_progress_by_instance(self, context, instance_uuid, migration_type=None): """Get all migrations of an instance in progress.""" diff --git a/nova/compute/migration_list.py b/nova/compute/migration_list.py new file mode 100644 index 0000000000..a2a7db6ae2 --- /dev/null +++ b/nova/compute/migration_list.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from nova.compute import multi_cell_list +from nova import context +from nova import db +from nova import exception +from nova import objects +from nova.objects import base + + +class MigrationSortContext(multi_cell_list.RecordSortContext): + def __init__(self, sort_keys, sort_dirs): + if not sort_keys: + sort_keys = ['created_at', 'id'] + sort_dirs = ['desc', 'desc'] + + if 'uuid' not in sort_keys: + # Add uuid into the list of sort_keys which Since we're striping + # across cell databases here, many sort_keys arrangements will + # yield nothing unique across all the databases to give us a stable + # ordering, which can mess up expected client pagination behavior. + # So, throw uuid into the sort_keys at the end if it's not already + # there to keep us repeatable. + sort_keys = copy.copy(sort_keys) + ['uuid'] + sort_dirs = copy.copy(sort_dirs) + ['asc'] + + super(MigrationSortContext, self).__init__(sort_keys, sort_dirs) + + +class MigrationLister(multi_cell_list.CrossCellLister): + def __init__(self, sort_keys, sort_dirs): + super(MigrationLister, self).__init__( + MigrationSortContext(sort_keys, sort_dirs)) + + @property + def marker_identifier(self): + return 'uuid' + + def get_marker_record(self, ctx, marker): + """Get the marker migration from its cell. + + This returns the marker migration from the cell in which it lives + """ + results = context.scatter_gather_skip_cell0( + ctx, db.migration_get_by_uuid, marker) + db_migration = None + for cell_uuid, result in results.items(): + if result not in (context.did_not_respond_sentinel, + context.raised_exception_sentinel): + db_migration = result + break + if not db_migration: + raise exception.MarkerNotFound(marker=marker) + return db_migration + + def get_marker_by_values(self, ctx, values): + return db.migration_get_by_sort_filters(ctx, + self.sort_ctx.sort_keys, + self.sort_ctx.sort_dirs, + values) + + def get_by_filters(self, ctx, filters, limit, marker, **kwargs): + return db.migration_get_all_by_filters( + ctx, filters, limit=limit, marker=marker, + sort_keys=self.sort_ctx.sort_keys, + sort_dirs=self.sort_ctx.sort_dirs) + + +def get_migration_objects_sorted(ctx, filters, limit, marker, + sort_keys, sort_dirs): + mig_generator = MigrationLister(sort_keys, sort_dirs).get_records_sorted( + ctx, filters, limit, marker) + return base.obj_make_list(ctx, objects.MigrationList(), objects.Migration, + mig_generator) diff --git a/nova/db/api.py b/nova/db/api.py index 69a521b189..b2cf4781c0 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -584,6 +584,17 @@ def migration_get_in_progress_by_instance(context, instance_uuid, migration_type) +def migration_get_by_sort_filters(context, sort_keys, sort_dirs, values): + """Get the uuid of the first migration in a sort order. + + Return the first migration (uuid) of the set where each column value + is greater than or equal to the matching one in @values, for each key + in @sort_keys. + """ + return IMPL.migration_get_by_sort_filters(context, sort_keys, sort_dirs, + values) + + #################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 13171f7fa4..190d579595 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2268,6 +2268,12 @@ def instance_get_by_sort_filters(context, sort_keys, sort_dirs, values): """ model = models.Instance + return _model_get_uuid_by_sort_filters(context, model, sort_keys, + sort_dirs, values) + + +def _model_get_uuid_by_sort_filters(context, model, sort_keys, sort_dirs, + values): query = context.session.query(model.uuid) # NOTE(danms): Below is a re-implementation of our @@ -4398,6 +4404,12 @@ def migration_get_all_by_filters(context, filters, return [] query = model_query(context, models.Migration) + if "uuid" in filters: + # The uuid filter is here for the MigrationLister and multi-cell + # paging support in the compute API. + uuid = filters["uuid"] + uuid = [uuid] if isinstance(uuid, six.string_types) else uuid + query = query.filter(models.Migration.uuid.in_(uuid)) if 'changes-since' in filters: changes_since = timeutils.normalize_time(filters['changes-since']) query = query. \ @@ -4441,6 +4453,20 @@ def migration_get_all_by_filters(context, filters, return query.all() +@require_context +@pick_context_manager_reader_allow_async +def migration_get_by_sort_filters(context, sort_keys, sort_dirs, values): + """Attempt to get a single migration based on a combination of sort + keys, directions and filter values. This is used to try to find a + marker migration when we don't have a marker uuid. + + This returns just a uuid of the migration that matched. + """ + model = models.Migration + return _model_get_uuid_by_sort_filters(context, model, sort_keys, + sort_dirs, values) + + @pick_context_manager_writer def migration_migrate_to_uuid(context, count): # Avoid circular import diff --git a/nova/tests/functional/compute/test_migration_list.py b/nova/tests/functional/compute/test_migration_list.py new file mode 100644 index 0000000000..d4fc7e17c8 --- /dev/null +++ b/nova/tests/functional/compute/test_migration_list.py @@ -0,0 +1,98 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +from nova.compute import migration_list +from nova import context +from nova import exception +from nova import objects +from nova import test +from nova.tests import uuidsentinel + + +class TestMigrationListObjects(test.TestCase): + NUMBER_OF_CELLS = 3 + + def setUp(self): + super(TestMigrationListObjects, self).setUp() + + self.context = context.RequestContext('fake', 'fake') + self.num_migrations = 3 + self.migrations = [] + + start = datetime.datetime(1985, 10, 25, 1, 21, 0) + + self.cells = objects.CellMappingList.get_all(self.context) + # Create three migrations in each of the real cells. Leave the + # first cell empty to make sure we don't break with an empty + # one. + for cell in self.cells[1:]: + for i in range(0, self.num_migrations): + with context.target_cell(self.context, cell) as cctx: + mig = objects.Migration(cctx, + uuid=getattr( + uuidsentinel, + '%s_mig%i' % (cell.name, i) + ), + created_at=start, + migration_type='resize', + instance_uuid=getattr( + uuidsentinel, + 'inst%i' % i) + ) + mig.create() + self.migrations.append(mig) + + def test_get_instance_objects_sorted(self): + filters = {} + limit = None + marker = None + sort_keys = ['uuid'] + sort_dirs = ['asc'] + migs = migration_list.get_migration_objects_sorted( + self.context, filters, limit, marker, + sort_keys, sort_dirs) + found_uuids = [x.uuid for x in migs] + had_uuids = sorted([x['uuid'] for x in self.migrations]) + self.assertEqual(had_uuids, found_uuids) + + def test_get_instance_objects_sorted_paged(self): + """Query a full first page and ensure an empty second one. + + This uses created_at which is enforced to be the same across + each migration by setUp(). This will help make sure we still + have a stable ordering, even when we only claim to care about + created_at. + """ + migp1 = migration_list.get_migration_objects_sorted( + self.context, {}, None, None, + ['created_at'], ['asc']) + self.assertEqual(len(self.migrations), len(migp1)) + migp2 = migration_list.get_migration_objects_sorted( + self.context, {}, None, migp1[-1]['uuid'], + ['created_at'], ['asc']) + self.assertEqual(0, len(migp2)) + + def test_get_marker_record_not_found(self): + marker = uuidsentinel.not_found + self.assertRaises(exception.MarkerNotFound, + migration_list.get_migration_objects_sorted, + self.context, {}, None, marker, None, None) + + def test_get_sorted_with_limit(self): + migs = migration_list.get_migration_objects_sorted( + self.context, {}, 2, None, ['uuid'], ['asc']) + uuids = [mig['uuid'] for mig in migs] + had_uuids = [mig.uuid for mig in self.migrations] + self.assertEqual(sorted(had_uuids)[:2], uuids) + self.assertEqual(2, len(uuids)) diff --git a/nova/tests/unit/db/test_db_api.py b/nova/tests/unit/db/test_db_api.py index 0af8cbdcff..6cab82d6ec 100644 --- a/nova/tests/unit/db/test_db_api.py +++ b/nova/tests/unit/db/test_db_api.py @@ -1571,6 +1571,14 @@ class MigrationTestCase(test.TestCase): hosts = [migration['source_compute'], migration['dest_compute']] self.assertIn(filters["host"], hosts) + def test_get_migrations_by_uuid_filters(self): + mig_uuid1 = self._create(uuid=uuidsentinel.mig_uuid1) + filters = {"uuid": [uuidsentinel.mig_uuid1]} + mig_get = db.migration_get_all_by_filters(self.ctxt, filters) + self.assertEqual(1, len(mig_get)) + for key in mig_uuid1: + self.assertEqual(mig_uuid1[key], mig_get[0][key]) + def test_get_migrations_by_filters_with_multiple_statuses(self): filters = {"status": ["reverted", "confirmed"], "migration_type": None, "hidden": False}