From fa002925460e70d988d1b4dd1ea594c680a43740 Mon Sep 17 00:00:00 2001 From: Andrea Rosa Date: Fri, 5 Feb 2016 08:31:06 +0000 Subject: [PATCH] Abort an ongoing live migration This change adds a DELETE call on the server-migrations object to cancel a running live migration of a specific instance. TO perform the cancellation the virtualization driver needs to support it, in case that the feature is not supported we return an error. We allow a cancellation of a migration only if the migration is running at the moment of the request and if the migration type is equal to 'live-migration'. In this change we implement this feature for the libvirt driver. When the cancellation of a live migration succeeded we rollback the live migration and we set the state of the Migration object equals to 'cancelled'. The implementation of this change is based on the work done by the implementation of the feature called 'force live migration': https://review.openstack.org/245921 DocImpact ApiImpact Implements blueprint: abort-live-migration Change-Id: I1ff861e54997a069894b542bd764ac3ef1b3dbb2 --- .../versions/v21-version-get-resp.json | 2 +- .../versions/versions-get-resp.json | 2 +- doc/notification_samples/service-update.json | 2 +- etc/nova/policy.json | 1 + nova/api/openstack/api_version_request.py | 4 +- .../openstack/compute/server_migrations.py | 19 +++++ .../openstack/rest_api_version_history.rst | 7 ++ nova/compute/api.py | 27 +++++++ nova/compute/instance_actions.py | 1 + nova/compute/manager.py | 34 +++++++- nova/compute/rpcapi.py | 8 ++ nova/objects/service.py | 4 +- .../test_server_migrations.py | 51 ++++++++++++ .../compute/test_server_migrations.py | 77 +++++++++++++++++++ nova/tests/unit/compute/test_compute.py | 24 ++++++ nova/tests/unit/compute/test_compute_api.py | 41 ++++++++++ nova/tests/unit/compute/test_compute_mgr.py | 62 +++++++++++++++ nova/tests/unit/compute/test_rpcapi.py | 5 ++ nova/tests/unit/fake_policy.py | 1 + nova/tests/unit/test_policy.py | 1 + nova/tests/unit/virt/libvirt/test_driver.py | 31 ++++++-- nova/tests/unit/virt/test_virt_drivers.py | 5 ++ nova/virt/driver.py | 8 ++ nova/virt/fake.py | 3 + nova/virt/libvirt/driver.py | 19 ++++- ...abort-live-migration-cb902bb0754b11b6.yaml | 5 ++ 26 files changed, 429 insertions(+), 15 deletions(-) create mode 100644 releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml diff --git a/doc/api_samples/versions/v21-version-get-resp.json b/doc/api_samples/versions/v21-version-get-resp.json index efbbf344ba..afb4e3dcc2 100644 --- a/doc/api_samples/versions/v21-version-get-resp.json +++ b/doc/api_samples/versions/v21-version-get-resp.json @@ -19,7 +19,7 @@ } ], "status": "CURRENT", - "version": "2.23", + "version": "2.24", "min_version": "2.1", "updated": "2013-07-23T11:33:21Z" } diff --git a/doc/api_samples/versions/versions-get-resp.json b/doc/api_samples/versions/versions-get-resp.json index 6b4eaada1a..1e34a553e7 100644 --- a/doc/api_samples/versions/versions-get-resp.json +++ b/doc/api_samples/versions/versions-get-resp.json @@ -22,7 +22,7 @@ } ], "status": "CURRENT", - "version": "2.23", + "version": "2.24", "min_version": "2.1", "updated": "2013-07-23T11:33:21Z" } diff --git a/doc/notification_samples/service-update.json b/doc/notification_samples/service-update.json index 219dec9ae2..f1e7e0bd9e 100644 --- a/doc/notification_samples/service-update.json +++ b/doc/notification_samples/service-update.json @@ -13,7 +13,7 @@ "disabled_reason": null, "report_count": 1, "forced_down": false, - "version": 7 + "version": 8 } }, "event_type": "service.update", diff --git a/etc/nova/policy.json b/etc/nova/policy.json index 3877c99edb..dc35f0c204 100644 --- a/etc/nova/policy.json +++ b/etc/nova/policy.json @@ -271,6 +271,7 @@ "os_compute_api:servers:stop": "rule:admin_or_owner", "os_compute_api:servers:trigger_crash_dump": "rule:admin_or_owner", "os_compute_api:servers:migrations:force_complete": "rule:admin_api", + "os_compute_api:servers:migrations:delete": "rule:admin_api", "os_compute_api:servers:discoverable": "@", "os_compute_api:servers:migrations:index": "rule:admin_api", "os_compute_api:servers:migrations:show": "rule:admin_api", diff --git a/nova/api/openstack/api_version_request.py b/nova/api/openstack/api_version_request.py index 77a358cc4b..33f364d999 100644 --- a/nova/api/openstack/api_version_request.py +++ b/nova/api/openstack/api_version_request.py @@ -68,6 +68,8 @@ REST_API_VERSION_HISTORY = """REST API Version History: * 2.23 - Add index/show API for server migrations. Also add migration_type for /os-migrations and add ref link for it when the migration is an in progress live migration. + * 2.24 - Add API to cancel a running live migration + """ # The minimum and maximum versions of the API supported @@ -76,7 +78,7 @@ REST_API_VERSION_HISTORY = """REST API Version History: # Note(cyeoh): This only applies for the v2.1 API once microversions # support is fully merged. It does not affect the V2 API. _MIN_API_VERSION = "2.1" -_MAX_API_VERSION = "2.23" +_MAX_API_VERSION = "2.24" DEFAULT_API_VERSION = _MIN_API_VERSION diff --git a/nova/api/openstack/compute/server_migrations.py b/nova/api/openstack/compute/server_migrations.py index 7acf6c4e4f..fa5ba5e5b5 100644 --- a/nova/api/openstack/compute/server_migrations.py +++ b/nova/api/openstack/compute/server_migrations.py @@ -135,6 +135,25 @@ class ServerMigrationsController(wsgi.Controller): return {'migration': output(migration)} + @wsgi.Controller.api_version("2.24") + @wsgi.response(202) + @extensions.expected_errors((400, 404, 409)) + def delete(self, req, server_id, id): + """Abort an in progress migration of an instance.""" + context = req.environ['nova.context'] + authorize(context, action="delete") + + instance = common.get_instance(self.compute_api, context, server_id) + try: + self.compute_api.live_migrate_abort(context, instance, id) + except exception.InstanceInvalidState as state_error: + common.raise_http_conflict_for_instance_invalid_state( + state_error, "abort live migration", server_id) + except exception.MigrationNotFoundForInstance as e: + raise exc.HTTPNotFound(explanation=e.format_message()) + except exception.InvalidMigrationState as e: + raise exc.HTTPBadRequest(explanation=e.format_message()) + class ServerMigrations(extensions.V21APIExtensionBase): """Server Migrations API.""" diff --git a/nova/api/openstack/rest_api_version_history.rst b/nova/api/openstack/rest_api_version_history.rst index 85c45087be..c363136c52 100644 --- a/nova/api/openstack/rest_api_version_history.rst +++ b/nova/api/openstack/rest_api_version_history.rst @@ -205,3 +205,10 @@ user documentation. Add migration_type for old /os-migrations API, also add ref link to the /servers/{uuid}/migrations/{id} for it when the migration is an in-progress live-migration. + +2.24 +--- + + A new API call to cancel a running live migration:: + + DELETE /servers//migrations/ diff --git a/nova/compute/api.py b/nova/compute/api.py index c4e0ee2218..a7b0d5e9f6 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -3328,6 +3328,33 @@ class API(base.Base): self.compute_rpcapi.live_migration_force_complete( context, instance, migration.id) + @check_instance_lock + @check_instance_cell + @check_instance_state(task_state=[task_states.MIGRATING]) + def live_migrate_abort(self, context, instance, migration_id): + """Abort an in-progress live migration. + + :param context: Security context + :param instance: The instance that is being migrated + :param migration_id: ID of in-progress live migration + + """ + migration = objects.Migration.get_by_id_and_instance(context, + migration_id, instance.uuid) + LOG.debug("Going to cancel live migration %s", + migration.id, instance=instance) + + if migration.status != 'running': + raise exception.InvalidMigrationState(migration_id=migration_id, + instance_uuid=instance.uuid, + state=migration.status, + method='abort live migration') + self._record_action_start(context, instance, + instance_actions.LIVE_MIGRATION_CANCEL) + + self.compute_rpcapi.live_migration_abort(context, + instance, migration.id) + @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED, vm_states.ERROR]) def evacuate(self, context, instance, host, on_shared_storage, diff --git a/nova/compute/instance_actions.py b/nova/compute/instance_actions.py index 1fbe7b1429..a08e660b98 100644 --- a/nova/compute/instance_actions.py +++ b/nova/compute/instance_actions.py @@ -49,4 +49,5 @@ CHANGE_PASSWORD = 'changePassword' SHELVE = 'shelve' UNSHELVE = 'unshelve' LIVE_MIGRATION = 'live-migration' +LIVE_MIGRATION_CANCEL = 'live_migration_cancel' TRIGGER_CRASH_DUMP = 'trigger_crash_dump' diff --git a/nova/compute/manager.py b/nova/compute/manager.py index a3d0669628..49d0317690 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -674,7 +674,7 @@ class ComputeVirtAPI(virtapi.VirtAPI): class ComputeManager(manager.Manager): """Manages the running instances from creation to destruction.""" - target = messaging.Target(version='4.9') + target = messaging.Target(version='4.10') # How long to wait in seconds before re-issuing a shutdown # signal to an instance during power off. The overall @@ -5278,6 +5278,30 @@ class ComputeManager(manager.Manager): self._notify_about_instance_usage( context, instance, 'live.migration.force.complete.end') + @wrap_exception() + @wrap_instance_event + @wrap_instance_fault + def live_migration_abort(self, context, instance, migration_id): + """Abort an in-progress live migration. + + :param context: Security context + :param instance: The instance that is being migrated + :param migration_id: ID of in-progress live migration + + """ + migration = objects.Migration.get_by_id(context, migration_id) + if migration.status != 'running': + raise exception.InvalidMigrationState(migration_id=migration_id, + instance_uuid=instance.uuid, + state=migration.status, + method='abort live migration') + + self._notify_about_instance_usage( + context, instance, 'live.migration.abort.start') + self.driver.live_migration_abort(instance) + self._notify_about_instance_usage( + context, instance, 'live.migration.abort.end') + def _live_migration_cleanup_flags(self, block_migration, migrate_data): """Determine whether disks or instance path need to be cleaned up after live migration (at source on success, at destination on rollback) @@ -5509,7 +5533,8 @@ class ComputeManager(manager.Manager): @wrap_exception() @wrap_instance_fault def _rollback_live_migration(self, context, instance, - dest, block_migration, migrate_data=None): + dest, block_migration, migrate_data=None, + migration_status='error'): """Recovers Instance/volume state from migrating -> running. :param context: security context @@ -5520,6 +5545,8 @@ class ComputeManager(manager.Manager): :param block_migration: if true, prepare for block migration :param migrate_data: if not none, contains implementation specific data. + :param migration_status: + Contains the status we want to set for the migration object """ instance.task_state = None @@ -5559,7 +5586,8 @@ class ComputeManager(manager.Manager): self._notify_about_instance_usage(context, instance, "live_migration._rollback.end") - self._set_migration_status(migration, 'error') + + self._set_migration_status(migration, migration_status) @wrap_exception() @wrap_instance_event diff --git a/nova/compute/rpcapi.py b/nova/compute/rpcapi.py index c9b33c6b3a..f78a307b52 100644 --- a/nova/compute/rpcapi.py +++ b/nova/compute/rpcapi.py @@ -326,6 +326,7 @@ class ComputeAPI(object): pre_live_migration. * ... - Remove refresh_provider_fw_rules() * 4.9 - Add live_migration_force_complete() + * 4.10 - Add live_migration_abort() ''' VERSION_ALIASES = { @@ -644,6 +645,13 @@ class ComputeAPI(object): cctxt.cast(ctxt, 'live_migration_force_complete', instance=instance, migration_id=migration_id) + def live_migration_abort(self, ctxt, instance, migration_id): + version = '4.10' + cctxt = self.client.prepare(server=_compute_host(None, instance), + version=version) + cctxt.cast(ctxt, 'live_migration_abort', instance=instance, + migration_id=migration_id) + def pause_instance(self, ctxt, instance): version = '4.0' cctxt = self.client.prepare(server=_compute_host(None, instance), diff --git a/nova/objects/service.py b/nova/objects/service.py index c7a5ee3ab9..15062a8d44 100644 --- a/nova/objects/service.py +++ b/nova/objects/service.py @@ -29,7 +29,7 @@ LOG = logging.getLogger(__name__) # NOTE(danms): This is the global service version counter -SERVICE_VERSION = 7 +SERVICE_VERSION = 8 # NOTE(danms): This is our SERVICE_VERSION history. The idea is that any @@ -67,6 +67,8 @@ SERVICE_VERSION_HISTORY = ( {'compute_rpc': '4.8'}, # Version 7: Add live_migration_force_complete in the compute_rpc {'compute_rpc': '4.9'}, + # Version 8: Add live_migration_abort in the compute_rpc + {'compute_rpc': '4.10'}, ) diff --git a/nova/tests/functional/api_sample_tests/test_server_migrations.py b/nova/tests/functional/api_sample_tests/test_server_migrations.py index 88d5279d72..a49743bde0 100644 --- a/nova/tests/functional/api_sample_tests/test_server_migrations.py +++ b/nova/tests/functional/api_sample_tests/test_server_migrations.py @@ -158,3 +158,54 @@ class ServerMigrationsSamplesJsonTestV2_23(test_servers.ServersSampleBase): self._verify_response('migrations-index', {"server_uuid_1": self.UUID_1}, response, 200) + + +class ServerMigrationsSampleJsonTestV2_24(test_servers.ServersSampleBase): + ADMIN_API = True + extension_name = "server-migrations" + scenarios = [('v2_24', {'api_major_version': 'v2.1'})] + extra_extensions_to_load = ["os-migrate-server", "os-access-ips"] + + def setUp(self): + """setUp method for server usage.""" + super(ServerMigrationsSampleJsonTestV2_24, self).setUp() + self.uuid = self._post_server() + self.context = context.RequestContext('fake', 'fake') + fake_migration = { + 'source_node': self.compute.host, + 'dest_node': 'node10', + 'source_compute': 'compute1', + 'dest_compute': 'compute12', + 'migration_type': 'live-migration', + 'instance_uuid': self.uuid, + 'status': 'running'} + + self.migration = objects.Migration(context=self.context, + **fake_migration) + self.migration.create() + + @mock.patch.object(conductor_manager.ComputeTaskManager, '_live_migrate') + def test_live_migrate_abort(self, _live_migrate): + self._do_post('servers/%s/action' % self.uuid, 'live-migrate-server', + {'hostname': self.compute.host}) + uri = 'servers/%s/migrations/%s' % (self.uuid, self.migration.id) + response = self._do_delete(uri, api_version='2.24') + self.assertEqual(202, response.status_code) + + @mock.patch.object(conductor_manager.ComputeTaskManager, '_live_migrate') + def test_live_migrate_abort_migration_not_found(self, _live_migrate): + self._do_post('servers/%s/action' % self.uuid, 'live-migrate-server', + {'hostname': self.compute.host}) + uri = 'servers/%s/migrations/%s' % (self.uuid, '45') + response = self._do_delete(uri, api_version='2.24') + self.assertEqual(404, response.status_code) + + @mock.patch.object(conductor_manager.ComputeTaskManager, '_live_migrate') + def test_live_migrate_abort_migration_not_running(self, _live_migrate): + self.migration.status = 'completed' + self.migration.save() + self._do_post('servers/%s/action' % self.uuid, 'live-migrate-server', + {'hostname': self.compute.host}) + uri = 'servers/%s/migrations/%s' % (self.uuid, self.migration.id) + response = self._do_delete(uri, api_version='2.24') + self.assertEqual(400, response.status_code) diff --git a/nova/tests/unit/api/openstack/compute/test_server_migrations.py b/nova/tests/unit/api/openstack/compute/test_server_migrations.py index a63b1d5dc2..6a18d6a72e 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_migrations.py +++ b/nova/tests/unit/api/openstack/compute/test_server_migrations.py @@ -261,6 +261,67 @@ class ServerMigrationsTestsV223(ServerMigrationsTestsV21): want_objects=True) +class ServerMigrationsTestsV224(ServerMigrationsTestsV21): + wsgi_api_version = '2.24' + + def setUp(self): + super(ServerMigrationsTestsV224, self).setUp() + self.req = fakes.HTTPRequest.blank('', version=self.wsgi_api_version, + use_admin_context=True) + self.context = self.req.environ['nova.context'] + + def test_cancel_live_migration_succeeded(self): + @mock.patch.object(self.compute_api, 'live_migrate_abort') + @mock.patch.object(self.compute_api, 'get') + def _do_test(mock_get, mock_abort): + self.controller.delete(self.req, 'server-id', 'migration-id') + mock_abort.assert_called_once_with(self.context, + mock_get(), + 'migration-id') + _do_test() + + def _test_cancel_live_migration_failed(self, fake_exc, expected_exc): + @mock.patch.object(self.compute_api, 'live_migrate_abort', + side_effect=fake_exc) + @mock.patch.object(self.compute_api, 'get') + def _do_test(mock_get, mock_abort): + self.assertRaises(expected_exc, + self.controller.delete, + self.req, + 'server-id', + 'migration-id') + _do_test() + + def test_cancel_live_migration_invalid_state(self): + self._test_cancel_live_migration_failed( + exception.InstanceInvalidState(instance_uuid='', + state='', + attr='', + method=''), + webob.exc.HTTPConflict) + + def test_cancel_live_migration_migration_not_found(self): + self._test_cancel_live_migration_failed( + exception.MigrationNotFoundForInstance(migration_id='', + instance_id=''), + webob.exc.HTTPNotFound) + + def test_cancel_live_migration_invalid_migration_state(self): + self._test_cancel_live_migration_failed( + exception.InvalidMigrationState(migration_id='', + instance_uuid='', + state='', + method=''), + webob.exc.HTTPBadRequest) + + def test_cancel_live_migration_instance_not_found(self): + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete, + self.req, + 'server-id', + 'migration-id') + + class ServerMigrationsPolicyEnforcementV21(test.NoDBTestCase): wsgi_api_version = '2.22' @@ -308,3 +369,19 @@ class ServerMigrationsPolicyEnforcementV223( fakes.FAKE_UUID, 1) self.assertEqual("Policy doesn't allow %s to be performed." % rule_name, exc.format_message()) + + +class ServerMigrationsPolicyEnforcementV224( + ServerMigrationsPolicyEnforcementV223): + + wsgi_api_version = '2.24' + + def setUp(self): + super(ServerMigrationsPolicyEnforcementV224, self).setUp() + + def test_migrate_delete_failed(self): + rule_name = "os_compute_api:servers:migrations:delete" + self.policy.set_rules({rule_name: "project:non_fake"}) + self.assertRaises(exception.PolicyNotAuthorized, + self.controller.delete, self.req, + fakes.FAKE_UUID, '10') diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index 0311b19f22..f644764218 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -5849,6 +5849,30 @@ class ComputeTestCase(BaseTestCase): self.assertEqual('error', migration.status) migration.save.assert_called_once_with() + @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid') + def test_rollback_live_migration_set_migration_status(self, mock_bdms): + c = context.get_admin_context() + instance = mock.MagicMock() + migration = mock.MagicMock() + migrate_data = {'migration': migration} + + mock_bdms.return_value = [] + + @mock.patch.object(self.compute, '_live_migration_cleanup_flags') + @mock.patch.object(self.compute, 'network_api') + def _test(mock_nw_api, mock_lmcf): + mock_lmcf.return_value = False, False + self.compute._rollback_live_migration(c, instance, 'foo', + False, + migrate_data=migrate_data, + migration_status='fake') + mock_nw_api.setup_networks_on_host.assert_called_once_with( + c, instance, self.compute.host) + _test() + + self.assertEqual('fake', migration.status) + migration.save.assert_called_once_with() + def test_rollback_live_migration_at_destination_correctly(self): # creating instance testdata c = context.get_admin_context() diff --git a/nova/tests/unit/compute/test_compute_api.py b/nova/tests/unit/compute/test_compute_api.py index 701dcca359..e7755284c8 100644 --- a/nova/tests/unit/compute/test_compute_api.py +++ b/nova/tests/unit/compute/test_compute_api.py @@ -3266,6 +3266,47 @@ class _ComputeAPIUnitTestMixIn(object): self.compute_api.live_migrate_force_complete, self.context, instance, '1') + def _get_migration(self, migration_id, status, migration_type): + migration = objects.Migration() + migration.id = migration_id + migration.status = status + migration.migration_type = migration_type + return migration + + @mock.patch('nova.compute.api.API._record_action_start') + @mock.patch.object(compute_rpcapi.ComputeAPI, 'live_migration_abort') + @mock.patch.object(objects.Migration, 'get_by_id_and_instance') + def test_live_migrate_abort_succeeded(self, + mock_get_migration, + mock_lm_abort, + mock_rec_action): + instance = self._create_instance_obj() + instance.task_state = task_states.MIGRATING + migration = self._get_migration(21, 'running', 'live-migration') + mock_get_migration.return_value = migration + + self.compute_api.live_migrate_abort(self.context, + instance, + migration.id) + mock_rec_action.assert_called_once_with(self.context, + instance, + instance_actions.LIVE_MIGRATION_CANCEL) + mock_lm_abort.called_once_with(self.context, instance, migration.id) + + @mock.patch.object(objects.Migration, 'get_by_id_and_instance') + def test_live_migration_abort_wrong_migration_status(self, + mock_get_migration): + instance = self._create_instance_obj() + instance.task_state = task_states.MIGRATING + migration = self._get_migration(21, 'completed', 'live-migration') + mock_get_migration.return_value = migration + + self.assertRaises(exception.InvalidMigrationState, + self.compute_api.live_migrate_abort, + self.context, + instance, + migration.id) + class ComputeAPIUnitTestCase(_ComputeAPIUnitTestMixIn, test.NoDBTestCase): def setUp(self): diff --git a/nova/tests/unit/compute/test_compute_mgr.py b/nova/tests/unit/compute/test_compute_mgr.py index 9b484959d2..17e812418d 100644 --- a/nova/tests/unit/compute/test_compute_mgr.py +++ b/nova/tests/unit/compute/test_compute_mgr.py @@ -4526,6 +4526,68 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase): _do_test() + def _get_migration(self, migration_id, status, migration_type): + migration = objects.Migration() + migration.id = migration_id + migration.status = status + migration.migration_type = migration_type + return migration + + @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage') + @mock.patch.object(objects.Migration, 'get_by_id') + @mock.patch.object(nova.virt.fake.SmallFakeDriver, 'live_migration_abort') + def test_live_migration_abort(self, + mock_driver, + mock_get_migration, + mock_notify): + instance = objects.Instance(id=123, uuid=uuids.instance) + migration = self._get_migration(10, 'running', 'live-migration') + mock_get_migration.return_value = migration + self.compute.live_migration_abort(self.context, instance, migration.id) + + mock_driver.assert_called_with(instance) + _notify_usage_calls = [mock.call(self.context, + instance, + 'live.migration.abort.start'), + mock.call(self.context, + instance, + 'live.migration.abort.end')] + + mock_notify.assert_has_calls(_notify_usage_calls) + + @mock.patch.object(compute_utils, 'add_instance_fault_from_exc') + @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage') + @mock.patch.object(objects.Migration, 'get_by_id') + @mock.patch.object(nova.virt.fake.SmallFakeDriver, 'live_migration_abort') + def test_live_migration_abort_not_supported(self, + mock_driver, + mock_get_migration, + mock_notify, + mock_instance_fault): + instance = objects.Instance(id=123, uuid=uuids.instance) + migration = self._get_migration(10, 'running', 'live-migration') + mock_get_migration.return_value = migration + mock_driver.side_effect = NotImplementedError() + self.assertRaises(NotImplementedError, + self.compute.live_migration_abort, + self.context, + instance, + migration.id) + + @mock.patch.object(compute_utils, 'add_instance_fault_from_exc') + @mock.patch.object(objects.Migration, 'get_by_id') + def test_live_migration_abort_wrong_migration_state(self, + mock_get_migration, + mock_instance_fault): + instance = objects.Instance(id=123, uuid=uuids.instance) + migration = self._get_migration(10, 'completed', 'live-migration') + mock_get_migration.return_value = migration + self.assertRaises(exception.InvalidMigrationState, + self.compute.live_migration_abort, + self.context, + instance, + migration.id) + class ComputeManagerInstanceUsageAuditTestCase(test.TestCase): def setUp(self): diff --git a/nova/tests/unit/compute/test_rpcapi.py b/nova/tests/unit/compute/test_rpcapi.py index 8396adef92..54af080d91 100644 --- a/nova/tests/unit/compute/test_rpcapi.py +++ b/nova/tests/unit/compute/test_rpcapi.py @@ -309,6 +309,11 @@ class ComputeRpcAPITestCase(test.NoDBTestCase): instance=self.fake_instance_obj, migration_id='1', version='4.9') + def test_live_migration_abort(self): + self._test_compute_api('live_migration_abort', 'cast', + instance=self.fake_instance_obj, + migration_id='1', version='4.10') + def test_post_live_migration_at_destination(self): self._test_compute_api('post_live_migration_at_destination', 'cast', instance=self.fake_instance_obj, diff --git a/nova/tests/unit/fake_policy.py b/nova/tests/unit/fake_policy.py index 42cae21400..540a3b164b 100644 --- a/nova/tests/unit/fake_policy.py +++ b/nova/tests/unit/fake_policy.py @@ -125,6 +125,7 @@ policy_data = """ "os_compute_api:servers:start": "", "os_compute_api:servers:stop": "", "os_compute_api:servers:trigger_crash_dump": "", + "os_compute_api:servers:migrations:delete": "rule:admin_api", "os_compute_api:servers:migrations:force_complete": "", "os_compute_api:servers:migrations:index": "rule:admin_api", "os_compute_api:servers:migrations:show": "rule:admin_api", diff --git a/nova/tests/unit/test_policy.py b/nova/tests/unit/test_policy.py index 1d8da7914e..998106a61f 100644 --- a/nova/tests/unit/test_policy.py +++ b/nova/tests/unit/test_policy.py @@ -301,6 +301,7 @@ class RealRolePolicyTestCase(test.NoDBTestCase): "os_compute_api:servers:index:get_all_tenants", "os_compute_api:servers:show:host_status", "os_compute_api:servers:migrations:force_complete", +"os_compute_api:servers:migrations:delete", "network:attach_external_network", "os_compute_api:os-admin-actions", "os_compute_api:os-admin-actions:reset_network", diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 0ab16cac0a..14364493f8 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -7677,7 +7677,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): mock_mig_save, mock_job_info, mock_sleep, - mock_time): + mock_time, + expected_mig_status=None): drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) instance = objects.Instance(**self.test_instance) dom = fakelibvirt.Domain(drvr._get_connection(), "", True) @@ -7746,8 +7747,13 @@ class LibvirtConnTestCase(test.NoDBTestCase): 'abortJob not called when failure expected') self.assertFalse(fake_post_method.called, 'Post method called when success not expected') - fake_recover_method.assert_called_once_with( - self.context, instance, dest, False, migrate_data) + if expected_mig_status: + fake_recover_method.assert_called_once_with( + self.context, instance, dest, False, migrate_data, + migration_status=expected_mig_status) + else: + fake_recover_method.assert_called_once_with( + self.context, instance, dest, False, migrate_data) def test_live_migration_monitor_success(self): # A normal sequence where see all the normal job states @@ -7847,7 +7853,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): ] self._test_live_migration_monitoring(domain_info_records, [], - self.EXPECT_FAILURE) + self.EXPECT_FAILURE, + expected_mig_status='cancelled') @mock.patch.object(fakelibvirt.virDomain, "migrateSetMaxDowntime") @mock.patch.object(libvirt_driver.LibvirtDriver, @@ -7929,7 +7936,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): ] self._test_live_migration_monitoring(domain_info_records, - fake_times, self.EXPECT_ABORT) + fake_times, self.EXPECT_ABORT, + expected_mig_status='cancelled') def test_live_migration_monitor_progress(self): self.flags(live_migration_completion_timeout=1000000, @@ -7960,7 +7968,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): ] self._test_live_migration_monitoring(domain_info_records, - fake_times, self.EXPECT_ABORT) + fake_times, self.EXPECT_ABORT, + expected_mig_status='cancelled') def test_live_migration_downtime_steps(self): self.flags(live_migration_downtime=400, group='libvirt') @@ -13329,6 +13338,16 @@ class LibvirtConnTestCase(test.NoDBTestCase): drvr.live_migration_force_complete(self.test_instance) pause.assert_called_once_with(self.test_instance) + @mock.patch.object(fakelibvirt.virDomain, "abortJob") + def test_live_migration_abort(self, mock_abort): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + dom = fakelibvirt.Domain(drvr._get_connection(), "", False) + guest = libvirt_guest.Guest(dom) + with mock.patch.object(nova.virt.libvirt.host.Host, 'get_guest', + return_value=guest): + drvr.live_migration_abort(self.test_instance) + self.assertTrue(mock_abort.called) + @mock.patch('os.path.exists', return_value=True) @mock.patch('tempfile.mkstemp') @mock.patch('os.close', return_value=None) diff --git a/nova/tests/unit/virt/test_virt_drivers.py b/nova/tests/unit/virt/test_virt_drivers.py index b592cc422f..78500aa81f 100644 --- a/nova/tests/unit/virt/test_virt_drivers.py +++ b/nova/tests/unit/virt/test_virt_drivers.py @@ -672,6 +672,11 @@ class _VirtDriverTestCase(_FakeDriverBackendTestCase): instance_ref, network_info = self._get_running_instance() self.connection.live_migration_force_complete(instance_ref) + @catch_notimplementederror + def test_live_migration_abort(self): + instance_ref, network_info = self._get_running_instance() + self.connection.live_migration_abort(instance_ref) + @catch_notimplementederror def _check_available_resource_fields(self, host_status): keys = ['vcpus', 'memory_mb', 'local_gb', 'vcpus_used', diff --git a/nova/virt/driver.py b/nova/virt/driver.py index 624b014907..6d5090e3e8 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -837,6 +837,14 @@ class ComputeDriver(object): """ raise NotImplementedError() + def live_migration_abort(self, instance): + """Abort an in-progress live migration. + + :param instance: instance that is live migrating + + """ + raise NotImplementedError() + def rollback_live_migration_at_destination(self, context, instance, network_info, block_device_info, diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 1c0e9cfe42..5b305492e7 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -471,6 +471,9 @@ class FakeDriver(driver.ComputeDriver): def live_migration_force_complete(self, instance): return + def live_migration_abort(self, instance): + return + def check_can_live_migrate_destination_cleanup(self, context, dest_check_data): return diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 488a33a2fb..e119cbc748 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -5815,6 +5815,23 @@ class LibvirtDriver(driver.ComputeDriver): post_method, recover_method, block_migration, migrate_data) + def live_migration_abort(self, instance): + """Aborting a running live-migration. + + :param instance: instance object that is in migration + + """ + + guest = self._host.get_guest(instance) + dom = guest._domain + + try: + dom.abortJob() + except libvirt.libvirtError as e: + LOG.error(_LE("Failed to cancel migration %s"), + e, instance=instance) + raise + def _update_xml(self, xml_str, migrate_bdm_info, listen_addrs, serial_listen_addr): xml_doc = etree.fromstring(xml_str) @@ -6428,7 +6445,7 @@ class LibvirtDriver(driver.ComputeDriver): LOG.warn(_LW("Migration operation was cancelled"), instance=instance) recover_method(context, instance, dest, block_migration, - migrate_data) + migrate_data, migration_status='cancelled') break else: LOG.warn(_LW("Unexpected migration job type: %d"), diff --git a/releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml b/releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml new file mode 100644 index 0000000000..f6e418ccf8 --- /dev/null +++ b/releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml @@ -0,0 +1,5 @@ +--- +features: + - A new REST API to cancel an ongoing live migration has been added + in microversion 2.24. Initially this operation will only work with + the libvirt virt driver.