Merge "libvirt: check for interface when detach_interface fails"

nova/tests/unit/virt/libvirt/test_driver.py

@@ -14781,6 +14781,30 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
             'detach_interface', power_state.SHUTDOWN,
             expected_flags=(fakelibvirt.VIR_DOMAIN_AFFECT_CONFIG))
 
+    @mock.patch('nova.virt.libvirt.driver.LOG')
+    def test_detach_interface_device_not_found(self, mock_log):
+        # Asserts that we don't log an error when the interface device is not
+        # found on the guest after a libvirt error during detach.
+        instance = self._create_instance()
+        vif = _fake_network_info(self, 1)[0]
+        guest = mock.Mock(spec='nova.virt.libvirt.guest.Guest')
+        guest.get_power_state = mock.Mock()
+        self.drvr._host.get_guest = mock.Mock(return_value=guest)
+        self.drvr.vif_driver = mock.Mock()
+        error = fakelibvirt.libvirtError(
+            'no matching network device was found')
+        error.err = (fakelibvirt.VIR_ERR_OPERATION_FAILED,)
+        guest.detach_device = mock.Mock(side_effect=error)
+        # mock out that get_interface_by_mac doesn't find the interface
+        guest.get_interface_by_mac = mock.Mock(return_value=None)
+        self.drvr.detach_interface(instance, vif)
+        guest.get_interface_by_mac.assert_called_once_with(vif['address'])
+        # an error shouldn't be logged, but a warning should be logged
+        self.assertFalse(mock_log.error.called)
+        self.assertEqual(1, mock_log.warning.call_count)
+        self.assertIn('the device is no longer found on the guest',
+                      six.text_type(mock_log.warning.call_args[0]))
+
     def test_rescue(self):
         instance = self._create_instance({'config_drive': None})
         dummyxml = ("<domain type='kvm'><name>instance-0000000a</name>"

nova/tests/unit/virt/libvirt/test_guest.py

@@ -411,6 +411,10 @@ class GuestTestCase(test.NoDBTestCase):
         self.assertEqual(1, len(devs))
         self.assertIsInstance(devs[0], vconfig.LibvirtConfigGuestInterface)
 
+        self.assertIsNotNone(
+            self.guest.get_interface_by_mac('fa:16:3e:f9:af:ae'))
+        self.assertIsNone(self.guest.get_interface_by_mac(None))
+
     def test_get_info(self):
         self.domain.info.return_value = (1, 2, 3, 4, 5)
         self.domain.ID.return_value = 6

nova/virt/libvirt/driver.py

@@ -1527,10 +1527,28 @@ class LibvirtDriver(driver.ComputeDriver):
                              "instance disappeared."),
                          instance=instance)
             else:
-                LOG.error(_LE('detaching network adapter failed.'),
-                         instance=instance, exc_info=True)
-                raise exception.InterfaceDetachFailed(
-                        instance_uuid=instance.uuid)
+                # NOTE(mriedem): When deleting an instance and using Neutron,
+                # we can be racing against Neutron deleting the port and
+                # sending the vif-deleted event which then triggers a call to
+                # detach the interface, so we might have failed because the
+                # network device no longer exists. Libvirt will fail with
+                # "operation failed: no matching network device was found"
+                # which unfortunately does not have a unique error code so we
+                # need to look up the interface by MAC and if it's not found
+                # then we can just log it as a warning rather than tracing an
+                # error.
+                mac = vif.get('address')
+                interface = guest.get_interface_by_mac(mac)
+                if interface:
+                    LOG.error(_LE('detaching network adapter failed.'),
+                             instance=instance, exc_info=True)
+                    raise exception.InterfaceDetachFailed(
+                            instance_uuid=instance.uuid)
+
+                # The interface is gone so just log it as a warning.
+                LOG.warning(_LW('Detaching interface %(mac)s failed  because '
+                                'the device is no longer found on the guest.'),
+                            {'mac': mac}, instance=instance)
 
     def _create_snapshot_metadata(self, image_meta, instance,
                                   img_fmt, snp_name):

nova/virt/libvirt/guest.py

@@ -186,6 +186,22 @@ class Guest(object):
 
         return interfaces
 
+    def get_interface_by_mac(self, mac):
+        """Lookup a LibvirtConfigGuestInterface by the MAC address.
+
+        :param mac: MAC address of the guest interface.
+        :type mac: str
+        :returns: nova.virt.libvirt.config.LibvirtConfigGuestInterface instance
+            if found, else None
+        """
+
+        if mac:
+            interfaces = self.get_all_devices(
+                vconfig.LibvirtConfigGuestInterface)
+            for interface in interfaces:
+                if interface.mac_addr == mac:
+                    return interface
+
     def get_vcpus_info(self):
         """Returns virtual cpus information of guest.