From a323ed29ee7fe5ebf040ad41cd770753a72d6397 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 29 Jun 2020 12:42:35 +0200 Subject: [PATCH] vmware: Handle exception in destroy with attached volumes When an attempt to delete an instance doesn't succeed and nova retries on the next nova-compute restart, an instance not existing in the back end anymore can lead to an uncatched exception in the vmware driver prohibiting instance deletion. This is the case, if the instance had volumes attached, because `_detach_instance_volumes()` always powers off the instance - which cannot work if the instance doesn't exist anymore. While the code already catched `ManagedObjectNotFoundException`, it also needs to catch `InstanceNotFound` raised by `vm_util.get_vm_ref()` to complete the deletion as seen in the traceback below (which comes from a queens codebase): Traceback (most recent call last): File "/nova/compute/manager.py", line 874, in _init_instance self._delete_instance(context, instance, bdms) File "/nova/hooks.py", line 154, in inner rv = f(*args, **kwargs) File "/nova/compute/manager.py", line 2500, in _delete_instance self._shutdown_instance(context, instance, bdms) File "/nova/compute/manager.py", line 2392, in _shutdown_instance requested_networks) File "/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__ self.force_reraise() File "/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise six.reraise(self.type_, self.value, self.tb) File "/nova/compute/manager.py", line 2379, in _shutdown_instance block_device_info) File "/nova/virt/vmwareapi/driver.py", line 574, in destroy self._detach_instance_volumes(instance, block_device_info) File "/nova/virt/vmwareapi/driver.py", line 536, in _detach_instance_volumes self._vmops.power_off(instance) File "/nova/virt/vmwareapi/vmops.py", line 1762, in power_off vm_util.power_off_instance(self._session, instance) File "/nova/virt/vmwareapi/vm_util.py", line 1732, in power_off_instance vm_ref = get_vm_ref(session, instance) File "/nova/virt/vmwareapi/vm_util.py", line 171, in wrapper return _vm_ref_cache(id, func, session, instance) File "/nova/virt/vmwareapi/vm_util.py", line 162, in _vm_ref_cache vm_ref = func(session, data) File "/nova/virt/vmwareapi/vm_util.py", line 1214, in get_vm_ref raise exception.InstanceNotFound(instance_id=uuid) InstanceNotFound: Instance 2af34cc5-22e0-400c-8b80-f130e86027fd could not be found. Change-Id: I65d2f76068e4b033ffd20959c9e74c870c8aa8e0 --- nova/tests/unit/virt/vmwareapi/test_driver_api.py | 11 +++++++++++ nova/virt/vmwareapi/driver.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index a0c20a9d46..375d29cf49 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -1593,6 +1593,17 @@ class VMwareAPIVMTestCase(test.NoDBTestCase, self._destroy_instance_without_vm_ref( task_state=task_states.RESIZE_REVERTING) + def test_destroy_instance_with_vm_ref_and_with_volumes(self): + self.destroy_disks = True + self._create_instance() + bdi = {'block_device_mapping': ['foo']} + with mock.patch.object(self.conn._vmops, + "destroy") as mock_destroy: + self.conn.destroy(self.context, self.instance, self.network_info, + bdi, self.destroy_disks) + mock_destroy.assert_called_once_with(self.instance, + self.destroy_disks) + def _rescue(self, config_drive=False): # validate that the power on is only called once self._power_on = vm_util.power_on_instance diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 852c3d33a5..fee9bcc700 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -623,7 +623,8 @@ class VMwareVCDriver(driver.ComputeDriver): if block_device_info is not None: try: self._detach_instance_volumes(instance, block_device_info) - except vexc.ManagedObjectNotFoundException: + except (vexc.ManagedObjectNotFoundException, + exception.InstanceNotFound): LOG.warning('Instance does not exists. Proceeding to ' 'delete instance properties on datastore', instance=instance)