From 8b33ac064456482158b23c2a2d52f819ebb4c60e Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Wed, 16 Dec 2020 13:12:13 +0000 Subject: [PATCH] only wait for plugtime events in pre-live-migration This change modifies _get_neutron_events_for_live_migration to filter the event to just the subset that will be sent at plug-time. Currently neuton has a bug where by the dhcp agent send a network-vif-plugged event during live migration after we update the port profile with "migrating-to:" this cause a network-vif-plugged event to be sent for configuration where vif_plugging in nova/os-vif is a noop. when that is corrected the current logic in nova cause the migration to time out as its waiting for an event that will never arrive. This change filters the set of events we wait for to just the plug time events. Related-Bug: #1815989 Closes-Bug: #1901707 Change-Id: Id2d8d72d30075200d2b07b847c4e5568599b0d3b --- nova/compute/manager.py | 9 +++--- nova/network/model.py | 23 ++++++++++--- nova/tests/unit/compute/test_compute_mgr.py | 36 +++++++++++++++++---- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 225ae3438f..6f7ccfe06c 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -8055,12 +8055,12 @@ class ComputeManager(manager.Manager): LOG.error(msg, msg_args) @staticmethod - def _get_neutron_events_for_live_migration(instance): + def _get_neutron_events_for_live_migration(instance, migration): # We don't generate events if CONF.vif_plugging_timeout=0 # meaning that the operator disabled using them. if CONF.vif_plugging_timeout: - return [('network-vif-plugged', vif['id']) - for vif in instance.get_network_info()] + return (instance.get_network_info() + .get_live_migration_plug_time_events()) else: return [] @@ -8131,7 +8131,8 @@ class ComputeManager(manager.Manager): """ pass - events = self._get_neutron_events_for_live_migration(instance) + events = self._get_neutron_events_for_live_migration( + instance, migration) try: if ('block_migration' in migrate_data and migrate_data.block_migration): diff --git a/nova/network/model.py b/nova/network/model.py index d8119fae72..7ed9d2d1b8 100644 --- a/nova/network/model.py +++ b/nova/network/model.py @@ -469,6 +469,14 @@ class VIF(Model): return (self.is_hybrid_plug_enabled() and not migration.is_same_host()) + @property + def has_live_migration_plug_time_event(self): + """Returns whether this VIF's network-vif-plugged external event will + be sent by Neutron at "plugtime" - in other words, as soon as neutron + completes configuring the network backend. + """ + return self.is_hybrid_plug_enabled() + def is_hybrid_plug_enabled(self): return self['details'].get(VIF_DETAILS_OVS_HYBRID_PLUG, False) @@ -530,15 +538,22 @@ class NetworkInfo(list): return jsonutils.dumps(self) def get_bind_time_events(self, migration): - """Returns whether any of our VIFs have "bind-time" events. See - has_bind_time_event() docstring for more details. + """Returns a list of external events for any VIFs that have + "bind-time" events during cold migration. """ return [('network-vif-plugged', vif['id']) for vif in self if vif.has_bind_time_event(migration)] + def get_live_migration_plug_time_events(self): + """Returns a list of external events for any VIFs that have + "plug-time" events during live migration. + """ + return [('network-vif-plugged', vif['id']) + for vif in self if vif.has_live_migration_plug_time_event] + def get_plug_time_events(self, migration): - """Complementary to get_bind_time_events(), any event that does not - fall in that category is a plug-time event. + """Returns a list of external events for any VIFs that have + "plug-time" events during cold migration. """ return [('network-vif-plugged', vif['id']) for vif in self if not vif.has_bind_time_event(migration)] diff --git a/nova/tests/unit/compute/test_compute_mgr.py b/nova/tests/unit/compute/test_compute_mgr.py index 1fd43ad474..9e555321d8 100644 --- a/nova/tests/unit/compute/test_compute_mgr.py +++ b/nova/tests/unit/compute/test_compute_mgr.py @@ -9256,24 +9256,41 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase, """Tests the various ways that _get_neutron_events_for_live_migration will return an empty list. """ + migration = mock.Mock() + migration.is_same_host = lambda: False + self.assertFalse(migration.is_same_host()) + # 1. no timeout self.flags(vif_plugging_timeout=0) with mock.patch.object(self.instance, 'get_network_info') as nw_info: nw_info.return_value = network_model.NetworkInfo( - [network_model.VIF(uuids.port1)]) + [network_model.VIF(uuids.port1, details={ + network_model.VIF_DETAILS_OVS_HYBRID_PLUG: True})]) + self.assertTrue(nw_info.return_value[0].is_hybrid_plug_enabled()) self.assertEqual( [], self.compute._get_neutron_events_for_live_migration( - self.instance)) + self.instance, migration)) # 2. no VIFs self.flags(vif_plugging_timeout=300) with mock.patch.object(self.instance, 'get_network_info') as nw_info: - nw_info.return_value = [] + nw_info.return_value = network_model.NetworkInfo([]) self.assertEqual( [], self.compute._get_neutron_events_for_live_migration( - self.instance)) + self.instance, migration)) + + # 3. no plug time events + with mock.patch.object(self.instance, 'get_network_info') as nw_info: + nw_info.return_value = network_model.NetworkInfo( + [network_model.VIF( + uuids.port1, details={ + network_model.VIF_DETAILS_OVS_HYBRID_PLUG: False})]) + self.assertFalse(nw_info.return_value[0].is_hybrid_plug_enabled()) + self.assertEqual( + [], self.compute._get_neutron_events_for_live_migration( + self.instance, migration)) @mock.patch('nova.compute.rpcapi.ComputeAPI.pre_live_migration') @mock.patch('nova.compute.manager.ComputeManager._post_live_migration') @@ -9288,9 +9305,11 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase, wait_for_vif_plugged=True) mock_get_bdms.return_value = objects.BlockDeviceMappingList(objects=[]) mock_pre_live_mig.return_value = migrate_data + details = {network_model.VIF_DETAILS_OVS_HYBRID_PLUG: True} self.instance.info_cache = objects.InstanceInfoCache( network_info=network_model.NetworkInfo([ - network_model.VIF(uuids.port1), network_model.VIF(uuids.port2) + network_model.VIF(uuids.port1, details=details), + network_model.VIF(uuids.port2, details=details) ])) self.compute._waiting_live_migrations[self.instance.uuid] = ( self.migration, mock.MagicMock() @@ -9320,11 +9339,12 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase, of not waiting. """ migrate_data = objects.LibvirtLiveMigrateData() + details = {network_model.VIF_DETAILS_OVS_HYBRID_PLUG: True} mock_get_bdms.return_value = objects.BlockDeviceMappingList(objects=[]) mock_pre_live_mig.return_value = migrate_data self.instance.info_cache = objects.InstanceInfoCache( network_info=network_model.NetworkInfo([ - network_model.VIF(uuids.port1)])) + network_model.VIF(uuids.port1, details=details)])) self.compute._waiting_live_migrations[self.instance.uuid] = ( self.migration, mock.MagicMock() ) @@ -9468,9 +9488,11 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase, mock_get_bdms.return_value = source_bdms migrate_data = objects.LibvirtLiveMigrateData( wait_for_vif_plugged=True) + details = {network_model.VIF_DETAILS_OVS_HYBRID_PLUG: True} self.instance.info_cache = objects.InstanceInfoCache( network_info=network_model.NetworkInfo([ - network_model.VIF(uuids.port1), network_model.VIF(uuids.port2) + network_model.VIF(uuids.port1, details=details), + network_model.VIF(uuids.port2, details=details) ])) self.compute._waiting_live_migrations = {} fake_migration = objects.Migration(