Merge "Add cross cell sort support for get_migrations"

nova/compute/api.py

@@ -44,6 +44,7 @@ from nova.cells import opts as cells_opts
 from nova.compute import flavors
 from nova.compute import instance_actions
 from nova.compute import instance_list
+from nova.compute import migration_list
 from nova.compute import power_state
 from nova.compute import rpcapi as compute_rpcapi
 from nova.compute import task_states
@@ -4250,6 +4251,13 @@ class API(base.Base):
                     cctxt, filters).objects)
         return objects.MigrationList(objects=migrations)
 
+    def get_migrations_sorted(self, context, filters, sort_dirs=None,
+                              sort_keys=None, limit=None, marker=None):
+        """Get all migrations for the given parameters."""
+        mig_objs = migration_list.get_migration_objects_sorted(
+            context, filters, limit, marker, sort_keys, sort_dirs)
+        return mig_objs
+
     def get_migrations_in_progress_by_instance(self, context, instance_uuid,
                                                migration_type=None):
         """Get all migrations of an instance in progress."""

nova/compute/migration_list.py

@@ -0,0 +1,86 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from nova.compute import multi_cell_list
+from nova import context
+from nova import db
+from nova import exception
+from nova import objects
+from nova.objects import base
+
+
+class MigrationSortContext(multi_cell_list.RecordSortContext):
+    def __init__(self, sort_keys, sort_dirs):
+        if not sort_keys:
+            sort_keys = ['created_at', 'id']
+            sort_dirs = ['desc', 'desc']
+
+        if 'uuid' not in sort_keys:
+            # Add uuid into the list of sort_keys which Since we're striping
+            # across cell databases here, many sort_keys arrangements will
+            # yield nothing unique across all the databases to give us a stable
+            # ordering, which can mess up expected client pagination behavior.
+            # So, throw uuid into the sort_keys at the end if it's not already
+            # there to keep us repeatable.
+            sort_keys = copy.copy(sort_keys) + ['uuid']
+            sort_dirs = copy.copy(sort_dirs) + ['asc']
+
+        super(MigrationSortContext, self).__init__(sort_keys, sort_dirs)
+
+
+class MigrationLister(multi_cell_list.CrossCellLister):
+    def __init__(self, sort_keys, sort_dirs):
+        super(MigrationLister, self).__init__(
+            MigrationSortContext(sort_keys, sort_dirs))
+
+    @property
+    def marker_identifier(self):
+        return 'uuid'
+
+    def get_marker_record(self, ctx, marker):
+        """Get the marker migration from its cell.
+
+        This returns the marker migration from the cell in which it lives
+        """
+        results = context.scatter_gather_skip_cell0(
+            ctx, db.migration_get_by_uuid, marker)
+        db_migration = None
+        for cell_uuid, result in results.items():
+            if result not in (context.did_not_respond_sentinel,
+                              context.raised_exception_sentinel):
+                db_migration = result
+                break
+        if not db_migration:
+            raise exception.MarkerNotFound(marker=marker)
+        return db_migration
+
+    def get_marker_by_values(self, ctx, values):
+        return db.migration_get_by_sort_filters(ctx,
+                                                self.sort_ctx.sort_keys,
+                                                self.sort_ctx.sort_dirs,
+                                                values)
+
+    def get_by_filters(self, ctx, filters, limit, marker, **kwargs):
+        return db.migration_get_all_by_filters(
+            ctx, filters, limit=limit, marker=marker,
+            sort_keys=self.sort_ctx.sort_keys,
+            sort_dirs=self.sort_ctx.sort_dirs)
+
+
+def get_migration_objects_sorted(ctx, filters, limit, marker,
+                                 sort_keys, sort_dirs):
+    mig_generator = MigrationLister(sort_keys, sort_dirs).get_records_sorted(
+        ctx, filters, limit, marker)
+    return base.obj_make_list(ctx, objects.MigrationList(), objects.Migration,
+                              mig_generator)

nova/db/api.py

@@ -584,6 +584,17 @@ def migration_get_in_progress_by_instance(context, instance_uuid,
                                                       migration_type)
 
 
+def migration_get_by_sort_filters(context, sort_keys, sort_dirs, values):
+    """Get the uuid of the first migration in a sort order.
+
+    Return the first migration (uuid) of the set where each column value
+    is greater than or equal to the matching one in @values, for each key
+    in @sort_keys.
+    """
+    return IMPL.migration_get_by_sort_filters(context, sort_keys, sort_dirs,
+                                              values)
+
+
 ####################
 
 

nova/db/sqlalchemy/api.py

@@ -2268,6 +2268,12 @@ def instance_get_by_sort_filters(context, sort_keys, sort_dirs, values):
     """
 
     model = models.Instance
+    return _model_get_uuid_by_sort_filters(context, model, sort_keys,
+                                           sort_dirs, values)
+
+
+def _model_get_uuid_by_sort_filters(context, model, sort_keys, sort_dirs,
+                                    values):
     query = context.session.query(model.uuid)
 
     # NOTE(danms): Below is a re-implementation of our
@@ -4398,6 +4404,12 @@ def migration_get_all_by_filters(context, filters,
         return []
 
     query = model_query(context, models.Migration)
+    if "uuid" in filters:
+        # The uuid filter is here for the MigrationLister and multi-cell
+        # paging support in the compute API.
+        uuid = filters["uuid"]
+        uuid = [uuid] if isinstance(uuid, six.string_types) else uuid
+        query = query.filter(models.Migration.uuid.in_(uuid))
     if 'changes-since' in filters:
         changes_since = timeutils.normalize_time(filters['changes-since'])
         query = query. \
@@ -4441,6 +4453,20 @@ def migration_get_all_by_filters(context, filters,
         return query.all()
 
 
+@require_context
+@pick_context_manager_reader_allow_async
+def migration_get_by_sort_filters(context, sort_keys, sort_dirs, values):
+    """Attempt to get a single migration based on a combination of sort
+    keys, directions and filter values. This is used to try to find a
+    marker migration when we don't have a marker uuid.
+
+    This returns just a uuid of the migration that matched.
+    """
+    model = models.Migration
+    return _model_get_uuid_by_sort_filters(context, model, sort_keys,
+                                           sort_dirs, values)
+
+
 @pick_context_manager_writer
 def migration_migrate_to_uuid(context, count):
     # Avoid circular import

nova/tests/functional/compute/test_migration_list.py

@@ -0,0 +1,98 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from nova.compute import migration_list
+from nova import context
+from nova import exception
+from nova import objects
+from nova import test
+from nova.tests import uuidsentinel
+
+
+class TestMigrationListObjects(test.TestCase):
+    NUMBER_OF_CELLS = 3
+
+    def setUp(self):
+        super(TestMigrationListObjects, self).setUp()
+
+        self.context = context.RequestContext('fake', 'fake')
+        self.num_migrations = 3
+        self.migrations = []
+
+        start = datetime.datetime(1985, 10, 25, 1, 21, 0)
+
+        self.cells = objects.CellMappingList.get_all(self.context)
+        # Create three migrations in each of the real cells. Leave the
+        # first cell empty to make sure we don't break with an empty
+        # one.
+        for cell in self.cells[1:]:
+            for i in range(0, self.num_migrations):
+                with context.target_cell(self.context, cell) as cctx:
+                    mig = objects.Migration(cctx,
+                                            uuid=getattr(
+                                                uuidsentinel,
+                                                '%s_mig%i' % (cell.name, i)
+                                            ),
+                                            created_at=start,
+                                            migration_type='resize',
+                                            instance_uuid=getattr(
+                                                uuidsentinel,
+                                                'inst%i' % i)
+                                            )
+                    mig.create()
+                self.migrations.append(mig)
+
+    def test_get_instance_objects_sorted(self):
+        filters = {}
+        limit = None
+        marker = None
+        sort_keys = ['uuid']
+        sort_dirs = ['asc']
+        migs = migration_list.get_migration_objects_sorted(
+            self.context, filters, limit, marker,
+            sort_keys, sort_dirs)
+        found_uuids = [x.uuid for x in migs]
+        had_uuids = sorted([x['uuid'] for x in self.migrations])
+        self.assertEqual(had_uuids, found_uuids)
+
+    def test_get_instance_objects_sorted_paged(self):
+        """Query a full first page and ensure an empty second one.
+
+        This uses created_at which is enforced to be the same across
+        each migration by setUp(). This will help make sure we still
+        have a stable ordering, even when we only claim to care about
+        created_at.
+        """
+        migp1 = migration_list.get_migration_objects_sorted(
+            self.context, {}, None, None,
+            ['created_at'], ['asc'])
+        self.assertEqual(len(self.migrations), len(migp1))
+        migp2 = migration_list.get_migration_objects_sorted(
+            self.context, {}, None, migp1[-1]['uuid'],
+            ['created_at'], ['asc'])
+        self.assertEqual(0, len(migp2))
+
+    def test_get_marker_record_not_found(self):
+        marker = uuidsentinel.not_found
+        self.assertRaises(exception.MarkerNotFound,
+                          migration_list.get_migration_objects_sorted,
+                          self.context, {}, None, marker, None, None)
+
+    def test_get_sorted_with_limit(self):
+        migs = migration_list.get_migration_objects_sorted(
+            self.context, {}, 2, None, ['uuid'], ['asc'])
+        uuids = [mig['uuid'] for mig in migs]
+        had_uuids = [mig.uuid for mig in self.migrations]
+        self.assertEqual(sorted(had_uuids)[:2], uuids)
+        self.assertEqual(2, len(uuids))

nova/tests/unit/db/test_db_api.py

@@ -1571,6 +1571,14 @@ class MigrationTestCase(test.TestCase):
             hosts = [migration['source_compute'], migration['dest_compute']]
             self.assertIn(filters["host"], hosts)
 
+    def test_get_migrations_by_uuid_filters(self):
+        mig_uuid1 = self._create(uuid=uuidsentinel.mig_uuid1)
+        filters = {"uuid": [uuidsentinel.mig_uuid1]}
+        mig_get = db.migration_get_all_by_filters(self.ctxt, filters)
+        self.assertEqual(1, len(mig_get))
+        for key in mig_uuid1:
+            self.assertEqual(mig_uuid1[key], mig_get[0][key])
+
     def test_get_migrations_by_filters_with_multiple_statuses(self):
         filters = {"status": ["reverted", "confirmed"],
                    "migration_type": None, "hidden": False}