Migrate network of an instance

In multi_host mode, floating ip(s) addr and NAT rules are still on
source node after resize/migrate instance. This patch fixes it up by
adding new methods in network.api to moving them to the destination
node.

Also adds the new methods to network/quantumv2/api.py. They do nothing
but pass for now.

This patch updates network RPC API to version 1.1

Fixes bug 1053344

Change-Id: If9f30050d37831f108ac4a1c8a018d820818f3b6

nova/compute/manager.py

@@ -1471,6 +1471,11 @@ class ComputeManager(manager.SchedulerDependentManager):
             self.network_api.setup_networks_on_host(context, instance,
                                                     teardown=True)
 
+            if migration_ref['dest_compute'] != \
+                                   migration_ref['source_compute']:
+                self.network_api.migrate_instance_start(context, instance,
+                                                migration_ref['dest_compute'])
+
             network_info = self._get_instance_nw_info(context, instance)
             block_device_info = self._get_instance_volume_block_device_info(
                                 context, instance['uuid'])
@@ -1532,6 +1537,11 @@ class ComputeManager(manager.SchedulerDependentManager):
                                        self._legacy_nw_info(network_info),
                                        block_device_info)
 
+            if migration_ref['dest_compute'] != \
+                                    migration_ref['source_compute']:
+                self.network_api.migrate_instance_finish(context, instance,
+                                            migration_ref['source_compute'])
+
             # Just roll back the record. There's no need to resize down since
             # the 'old' VM already has the preferred attributes
             self._instance_update(context,
@@ -1659,6 +1669,11 @@ class ComputeManager(manager.SchedulerDependentManager):
                     self.volume_api.terminate_connection(context, volume,
                             connector)
 
+            if migration_ref['dest_compute'] != \
+                                    migration_ref['source_compute']:
+                self.network_api.migrate_instance_start(context, instance,
+                                                           self.host)
+
             self.db.migration_update(context,
                                      migration_id,
                                      {'status': 'post-migrating'})
@@ -1697,6 +1712,11 @@ class ComputeManager(manager.SchedulerDependentManager):
         self.network_api.setup_networks_on_host(context, instance,
                                                 migration_ref['dest_compute'])
 
+        if migration_ref['dest_compute'] != \
+                                    migration_ref['source_compute']:
+            self.network_api.migrate_instance_finish(context, instance,
+                                                migration_ref['dest_compute'])
+
         network_info = self._get_instance_nw_info(context, instance)
 
         self._instance_update(context, instance['uuid'],

nova/db/api.py

@@ -641,6 +641,11 @@ def instance_get_floating_address(context, instance_id):
     return IMPL.instance_get_floating_address(context, instance_id)
 
 
+def instance_floating_address_get_all(context, instance_uuid):
+    """Get all floating ip addresses of an instance"""
+    return IMPL.instance_floating_address_get_all(context, instance_uuid)
+
+
 def instance_get_all_hung_in_rebooting(context, reboot_window):
     """Get all instances stuck in a rebooting state."""
     return IMPL.instance_get_all_hung_in_rebooting(context, reboot_window)

nova/db/sqlalchemy/api.py

@@ -1738,6 +1738,19 @@ def instance_get_floating_address(context, instance_id):
     return floating_ips[0]['address']
 
 
+@require_context
+def instance_floating_address_get_all(context, instance_uuid):
+    fixed_ips = fixed_ip_get_by_instance(context, instance_uuid)
+
+    floating_ips = []
+    for fixed_ip in fixed_ips:
+        _floating_ips = floating_ip_get_by_fixed_ip_id(context,
+                                                    fixed_ip['id'])
+        floating_ips += _floating_ips
+
+    return floating_ips
+
+
 @require_admin_context
 def instance_get_all_hung_in_rebooting(context, reboot_window, session=None):
     reboot_window = (timeutils.utcnow() -

nova/network/api.py

@@ -320,4 +320,37 @@ class API(base.Base):
         args = {'instance_id': instance['id'],
                 'host': host,
                 'teardown': teardown}
+
         self.network_rpcapi.setup_networks_on_host(context, **args)
+
+    def _is_multi_host(self, context, instance):
+        fixed_ips = self.db.fixed_ip_get_by_instance(context, instance['uuid'])
+
+        network = self.db.network_get(context, fixed_ips[0]['network_id'],
+                                      project_only=True)
+        return network['multi_host']
+
+    def _get_floating_ip_addresses(self, context, instance):
+        floating_ips = self.db.instance_floating_address_get_all(context,
+                                                            instance['uuid'])
+        return [floating_ip['address'] for floating_ip in floating_ips]
+
+    def migrate_instance_start(self, context, instance, host):
+        """Start to migrate the network of an instance"""
+        if self._is_multi_host(context, instance):
+            addresses = self._get_floating_ip_addresses(context, instance)
+            if addresses:
+                self.network_rpcapi.migrate_instance_start(context,
+                                                           instance['uuid'],
+                                                           addresses,
+                                                           host)
+
+    def migrate_instance_finish(self, context, instance, dest):
+        """Finish migrating the network of an instance"""
+        if self._is_multi_host(context, instance):
+            addresses = self._get_floating_ip_addresses(context, instance)
+            if addresses:
+                self.network_rpcapi.migrate_instance_finish(context,
+                                                            instance['uuid'],
+                                                            addresses,
+                                                            dest)

nova/network/manager.py

@@ -636,6 +636,68 @@ class FloatingIP(object):
                                                                 fixed_address)
         return [floating_ip['address'] for floating_ip in floating_ips]
 
+    def _is_stale_floating_ip_address(self, context, floating_ip):
+        try:
+            self._floating_ip_owned_by_project(context, floating_ip)
+        except exception.NotAuthorized:
+            return True
+        return False if floating_ip.get('fixed_ip_id') else True
+
+    @wrap_check_policy
+    def migrate_instance_start(self, context, instance_uuid,
+                               floating_addresses):
+        LOG.info(_("Starting migration network for instance"
+                   " %(instance_uuid)s"), locals())
+        for address in floating_addresses:
+            floating_ip = self.db.floating_ip_get_by_address(context,
+                                                             address)
+
+            if self._is_stale_floating_ip_address(context, floating_ip):
+                LOG.warn(_("Floating ip address |%(address)s| no longer "
+                           "belongs to instance %(instance_uuid)s. Will not"
+                           "migrate it "), locals())
+                continue
+
+            interface = FLAGS.public_interface or floating_ip['interface']
+            fixed_ip = self.db.fixed_ip_get(context,
+                                            floating_ip['fixed_ip_id'])
+            self.l3driver.remove_floating_ip(floating_ip['address'],
+                                             fixed_ip['address'],
+                                             interface)
+
+            # NOTE(wenjianhn): Make this address will not be bound to public
+            # interface when restarts nova-network on dest compute node
+            self.db.floating_ip_update(context,
+                                       floating_ip['address'],
+                                       {'host': None})
+
+    @wrap_check_policy
+    def migrate_instance_finish(self, context, instance_uuid,
+                                floating_addresses, host):
+        LOG.info(_("Finishing migration network for instance"
+                   " %(instance_uuid)s"), locals())
+
+        for address in floating_addresses:
+            floating_ip = self.db.floating_ip_get_by_address(context,
+                                                             address)
+
+            if self._is_stale_floating_ip_address(context, floating_ip):
+                LOG.warn(_("Floating ip address |%(address)s| no longer "
+                           "belongs to instance %(instance_uuid)s. Will not"
+                           "setup it."), locals())
+                continue
+
+            self.db.floating_ip_update(context,
+                                       floating_ip['address'],
+                                       {'host': host})
+
+            interface = FLAGS.public_interface or floating_ip['interface']
+            fixed_ip = self.db.fixed_ip_get(context,
+                                            floating_ip['fixed_ip_id'])
+            self.l3driver.add_floating_ip(floating_ip['address'],
+                                          fixed_ip['address'],
+                                          interface)
+
     def _prepare_domain_entry(self, context, domain):
         domainref = self.db.dnsdomain_get(context, domain)
         scope = domainref.scope
@@ -749,7 +811,7 @@ class NetworkManager(manager.SchedulerDependentManager):
         The one at a time part is to flatten the layout to help scale
     """
 
-    RPC_API_VERSION = '1.0'
+    RPC_API_VERSION = '1.1'
 
     # If True, this manager requires VIF to create a bridge.
     SHOULD_CREATE_BRIDGE = False

nova/network/quantumv2/api.py

@@ -494,6 +494,18 @@ class API(base.Base):
         fip = self._get_floating_ip_by_address(client, address)
         client.update_floatingip(fip['id'], {'floatingip': {'port_id': None}})
 
+    def migrate_instance_start(self, context, instance, host):
+        """Start to migrate the network of an instance"""
+        # NOTE(wenjianhn): just pass to make migrate instance doesn't
+        # raise for now.
+        pass
+
+    def migrate_instance_finish(self, context, instance, dest):
+        """Finish migrating the network of an instance"""
+        # NOTE(wenjianhn): just pass to make migrate instance doesn't
+        # raise for now.
+        pass
+
     def add_network_to_project(self, context, project_id, network_uuid=None):
         """Force add a network to the project."""
         raise NotImplementedError()

nova/network/rpcapi.py

@@ -33,6 +33,7 @@ class NetworkAPI(rpc_proxy.RpcProxy):
     API version history:
 
         1.0 - Initial version.
+        1.1 - Adds migrate_instance_[start|finish]
     '''
 
     #
@@ -264,3 +265,18 @@ class NetworkAPI(rpc_proxy.RpcProxy):
         return self.cast(ctxt, self.make_msg('release_fixed_ip',
                 address=address),
                 topic=rpc.queue_get_for(ctxt, self.topic, host))
+
+    def migrate_instance_start(self, ctxt, instance_uuid,
+                               floating_addresses, host):
+        return self.call(ctxt, self.make_msg('migrate_instance_start',
+                                             instance_uuid=instance_uuid,
+                                    floating_addresses=floating_addresses),
+                         topic=rpc.queue_get_for(ctxt, self.topic, host))
+
+    def migrate_instance_finish(self, ctxt, instance_uuid,
+                                floating_addresses, dest):
+        return self.call(ctxt, self.make_msg('migrate_instance_finish',
+                                             instance_uuid=instance_uuid,
+                                    floating_addresses=floating_addresses,
+                                             host=dest),
+                         topic=rpc.queue_get_for(ctxt, self.topic, dest))

nova/tests/network/test_manager.py

@@ -1627,6 +1627,79 @@ class FloatingIPTestCase(test.TestCase):
         self.network.deallocate_for_instance(self.context,
                 instance_id=instance['id'])
 
+    def test_migrate_instance_start(self):
+        called = {'count': 0}
+
+        def fake_floating_ip_get_by_address(context, address):
+            return {'address': address,
+                    'fixed_ip_id': 0}
+
+        def fake_is_stale_floating_ip_address(context, floating_ip):
+            return floating_ip['address'] == '172.24.4.23'
+
+        def fake_fixed_ip_get(context, fixed_ip_id):
+            return {'instance_uuid': 'fake_uuid',
+                    'address': '10.0.0.2'}
+
+        def fake_remove_floating_ip(floating_addr, fixed_addr, interface):
+            called['count'] += 1
+
+        def fake_floating_ip_update(context, address, args):
+            pass
+
+        self.stubs.Set(self.network.db, 'floating_ip_get_by_address',
+                       fake_floating_ip_get_by_address)
+        self.stubs.Set(self.network, '_is_stale_floating_ip_address',
+                                 fake_is_stale_floating_ip_address)
+        self.stubs.Set(self.network.db, 'fixed_ip_get', fake_fixed_ip_get)
+        self.stubs.Set(self.network.db, 'floating_ip_update',
+                       fake_floating_ip_update)
+        self.stubs.Set(self.network.l3driver, 'remove_floating_ip',
+                       fake_remove_floating_ip)
+        self.mox.ReplayAll()
+        floating_ip_addresses = ['172.24.4.23', '172.24.4.24', '172.24.4.25']
+        self.network.migrate_instance_start(self.context, FAKEUUID,
+                                               floating_ip_addresses)
+
+        self.assertEqual(called['count'], 2)
+
+    def test_migrate_instance_finish(self):
+        called = {'count': 0}
+
+        def fake_floating_ip_get_by_address(context, address):
+            return {'address': address,
+                    'fixed_ip_id': 0}
+
+        def fake_is_stale_floating_ip_address(context, floating_ip):
+            return floating_ip['address'] == '172.24.4.23'
+
+        def fake_fixed_ip_get(context, fixed_ip_id):
+            return {'instance_uuid': 'fake_uuid',
+                    'address': '10.0.0.2'}
+
+        def fake_add_floating_ip(floating_addr, fixed_addr, interface):
+            called['count'] += 1
+
+        def fake_floating_ip_update(context, address, args):
+            pass
+
+        self.stubs.Set(self.network.db, 'floating_ip_get_by_address',
+                       fake_floating_ip_get_by_address)
+        self.stubs.Set(self.network, '_is_stale_floating_ip_address',
+                                 fake_is_stale_floating_ip_address)
+        self.stubs.Set(self.network.db, 'fixed_ip_get', fake_fixed_ip_get)
+        self.stubs.Set(self.network.db, 'floating_ip_update',
+                       fake_floating_ip_update)
+        self.stubs.Set(self.network.l3driver, 'add_floating_ip',
+                       fake_add_floating_ip)
+        self.mox.ReplayAll()
+        floating_ip_addresses = ['172.24.4.23', '172.24.4.24', '172.24.4.25']
+        self.network.migrate_instance_finish(self.context, FAKEUUID,
+                                               floating_ip_addresses,
+                                               'fake_dest')
+
+        self.assertEqual(called['count'], 2)
+
     def test_floating_dns_create_conflict(self):
         zone = "example.org"
         address1 = "10.10.10.11"

nova/tests/policy.json

@@ -178,6 +178,8 @@
     "network:deallocate_floating_ip": "",
     "network:associate_floating_ip": "",
     "network:disassociate_floating_ip": "",
+    "network:migrate_instance_start": "",
+    "network:migrate_instance_finish": "",
 
     "network:get_fixed_ip": "",
     "network:get_fixed_ip_by_address": "",