womax/nova
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "XenAPI:Resolve Nova/Neutron race condition"

Browse Source
This commit is contained in:
Jenkins
2016-03-11 11:23:59 +00:00
committed by Gerrit Code Review
parent 59a07f00ad ecd3eb7c94
commit dae13c5153
3 changed files with 188 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nova/tests/unit/virt/xenapi/test_vmops.py
+118 -32
View File
@@ -33,6 +33,7 @@ from nova import test
from nova.tests.unit import fake_flavor
from nova.tests.unit import fake_instance
from nova.tests.unit.virt.xenapi import stubs
from nova import utils
from nova.virt import fake
from nova.virt.xenapi import agent as xenapi_agent
from nova.virt.xenapi.client import session as xenapi_session
@@ -322,10 +323,11 @@ class SpawnTestCase(VMOpsTestBase):
self.mox.StubOutWithMock(self.vmops.firewall_driver,
'apply_instance_filter')
self.mox.StubOutWithMock(self.vmops, '_update_last_dom_id')
self.mox.StubOutWithMock(self.vmops._session, 'call_xenapi')
def _test_spawn(self, name_label_param=None, block_device_info_param=None,
rescue=False, include_root_vdi=True, throw_exception=None,
attach_pci_dev=False):
attach_pci_dev=False, neutron_exception=False):
self._stub_out_common()
instance = {"name": "dummy", "uuid": "fake_uuid"}
@@ -418,38 +420,55 @@ class SpawnTestCase(VMOpsTestBase):
step += 1
self.vmops._update_instance_progress(context, instance, step, steps)
self.vmops._create_vifs(instance, vm_ref, network_info)
self.vmops.firewall_driver.setup_basic_filtering(instance,
network_info).AndRaise(NotImplementedError)
self.vmops.firewall_driver.prepare_instance_filter(instance,
network_info)
step += 1
self.vmops._update_instance_progress(context, instance, step, steps)
if rescue:
self.vmops._attach_orig_disks(instance, vm_ref)
if neutron_exception:
events = [('network-vif-plugged', 1)]
self.vmops._get_neutron_events(network_info,
True, True).AndReturn(events)
self.mox.StubOutWithMock(self.vmops, '_neutron_failed_callback')
self.mox.StubOutWithMock(self.vmops._virtapi,
'wait_for_instance_event')
self.vmops._virtapi.wait_for_instance_event(instance, events,
deadline=300,
error_callback=self.vmops._neutron_failed_callback).\
AndRaise(exception.VirtualInterfaceCreateException)
else:
self.vmops._create_vifs(instance, vm_ref, network_info)
self.vmops.firewall_driver.setup_basic_filtering(instance,
network_info).AndRaise(NotImplementedError)
self.vmops.firewall_driver.prepare_instance_filter(instance,
network_info)
step += 1
self.vmops._update_instance_progress(context, instance, step,
steps)
self.vmops._start(instance, vm_ref)
self.vmops._wait_for_instance_to_start(instance, vm_ref)
self.vmops._update_last_dom_id(vm_ref)
step += 1
self.vmops._update_instance_progress(context, instance, step, steps)
self.vmops._update_instance_progress(context, instance,
step, steps)
self.vmops._configure_new_instance_with_agent(instance, vm_ref,
injected_files, admin_password)
self.vmops._remove_hostname(instance, vm_ref)
step += 1
self.vmops._update_instance_progress(context, instance, step, steps)
if rescue:
self.vmops._attach_orig_disks(instance, vm_ref)
step += 1
self.vmops._update_instance_progress(context, instance, step,
steps)
start_pause = True
self.vmops._start(instance, vm_ref, start_pause=start_pause)
step += 1
self.vmops._update_instance_progress(context, instance,
step, steps)
self.vmops.firewall_driver.apply_instance_filter(instance,
network_info)
step += 1
self.vmops._update_instance_progress(context, instance,
step, steps)
self.vmops._session.call_xenapi('VM.unpause', vm_ref)
self.vmops._wait_for_instance_to_start(instance, vm_ref)
self.vmops._update_last_dom_id(vm_ref)
self.vmops._configure_new_instance_with_agent(instance, vm_ref,
injected_files, admin_password)
self.vmops._remove_hostname(instance, vm_ref)
step += 1
last_call = self.vmops._update_instance_progress(context, instance,
step, steps)
self.vmops.firewall_driver.apply_instance_filter(instance,
network_info)
step += 1
last_call = self.vmops._update_instance_progress(context, instance,
step, steps)
if throw_exception:
last_call.AndRaise(throw_exception)
if throw_exception or neutron_exception:
self.vmops._destroy(instance, vm_ref, network_info=network_info)
vm_utils.destroy_kernel_ramdisk(self.vmops._session, instance,
kernel_file, ramdisk_file)
@@ -476,11 +495,25 @@ class SpawnTestCase(VMOpsTestBase):
self.assertRaises(test.TestingException, self._test_spawn,
throw_exception=test.TestingException())
def test_spawn_with_neutron(self):
self.mox.StubOutWithMock(self.vmops, '_get_neutron_events')
events = [('network-vif-plugged', 1)]
network_info = "net_info"
self.vmops._get_neutron_events(network_info,
True, True).AndReturn(events)
self.mox.StubOutWithMock(self.vmops,
'_neutron_failed_callback')
self._test_spawn()
def test_spawn_with_neutron_exception(self):
self.mox.StubOutWithMock(self.vmops, '_get_neutron_events')
self.assertRaises(exception.VirtualInterfaceCreateException,
self._test_spawn, neutron_exception=True)
def _test_finish_migration(self, power_on=True, resize_instance=True,
throw_exception=None, booted_from_volume=False):
self._stub_out_common()
self.mox.StubOutWithMock(volumeops.VolumeOps, "connect_volume")
self.mox.StubOutWithMock(self.vmops._session, 'call_xenapi')
self.mox.StubOutWithMock(vm_utils, "import_all_migrated_disks")
self.mox.StubOutWithMock(self.vmops, "_attach_mapped_block_devices")
@@ -548,12 +581,14 @@ class SpawnTestCase(VMOpsTestBase):
network_info)
if power_on:
self.vmops._start(instance, vm_ref)
self.vmops._wait_for_instance_to_start(instance, vm_ref)
self.vmops._update_last_dom_id(vm_ref)
self.vmops._start(instance, vm_ref, start_pause=True)
self.vmops.firewall_driver.apply_instance_filter(instance,
network_info)
if power_on:
self.vmops._session.call_xenapi('VM.unpause', vm_ref)
self.vmops._wait_for_instance_to_start(instance, vm_ref)
self.vmops._update_last_dom_id(vm_ref)
last_call = self.vmops._update_instance_progress(context, instance,
step=5, total_steps=5)
@@ -711,6 +746,57 @@ class SpawnTestCase(VMOpsTestBase):
self.vmops._configure_new_instance_with_agent(instance, vm_ref,
None, None)
@mock.patch.object(utils, 'is_neutron', return_value=True)
def test_get_neutron_event(self, mock_is_neutron):
network_info = [{"active": False, "id": 1},
{"active": True, "id": 2},
{"active": False, "id": 3},
{"id": 4}]
power_on = True
first_boot = True
events = self.vmops._get_neutron_events(network_info,
power_on, first_boot)
self.assertEqual("network-vif-plugged", events[0][0])
self.assertEqual(1, events[0][1])
self.assertEqual("network-vif-plugged", events[1][0])
self.assertEqual(3, events[1][1])
@mock.patch.object(utils, 'is_neutron', return_value=False)
def test_get_neutron_event_not_neutron_network(self, mock_is_neutron):
network_info = [{"active": False, "id": 1},
{"active": True, "id": 2},
{"active": False, "id": 3},
{"id": 4}]
power_on = True
first_boot = True
events = self.vmops._get_neutron_events(network_info,
power_on, first_boot)
self.assertEqual([], events)
@mock.patch.object(utils, 'is_neutron', return_value=True)
def test_get_neutron_event_power_off(self, mock_is_neutron):
network_info = [{"active": False, "id": 1},
{"active": True, "id": 2},
{"active": False, "id": 3},
{"id": 4}]
power_on = False
first_boot = True
events = self.vmops._get_neutron_events(network_info,
power_on, first_boot)
self.assertEqual([], events)
@mock.patch.object(utils, 'is_neutron', return_value=True)
def test_get_neutron_event_not_first_boot(self, mock_is_neutron):
network_info = [{"active": False, "id": 1},
{"active": True, "id": 2},
{"active": False, "id": 3},
{"id": 4}]
power_on = True
first_boot = False
events = self.vmops._get_neutron_events(network_info,
power_on, first_boot)
self.assertEqual([], events)
class DestroyTestCase(VMOpsTestBase):
def setUp(self):
nova/tests/unit/virt/xenapi/test_xenapi.py
+10
View File
@@ -326,6 +326,11 @@ class XenAPIVMTestCase(stubs.XenAPITestBase):
virtual_size)
self.stubs.Set(vm_utils, '_safe_copy_vdi', fake_safe_copy_vdi)
def fake_unpause_and_wait(self, vm_ref, instance, power_on):
self._update_last_dom_id(vm_ref)
self.stubs.Set(vmops.VMOps, '_unpause_and_wait',
fake_unpause_and_wait)
def tearDown(self):
fake_image.FakeImageService_reset()
super(XenAPIVMTestCase, self).tearDown()
@@ -1689,6 +1694,11 @@ class XenAPIMigrateInstance(stubs.XenAPITestBase):
self.stubs.Set(vmops.VMOps, '_inject_instance_metadata',
fake_inject_instance_metadata)
def fake_unpause_and_wait(self, vm_ref, instance, power_on):
pass
self.stubs.Set(vmops.VMOps, '_unpause_and_wait',
fake_unpause_and_wait)
def _create_instance(self, **kw):
values = self.instance_values.copy()
values.update(kw)
nova/virt/xenapi/vmops.py
+60 -14
View File
@@ -22,6 +22,7 @@ import functools
import time
import zlib
import eventlet
from eventlet import greenthread
import netaddr
from oslo_config import cfg
@@ -321,7 +322,8 @@ class VMOps(object):
rescue=False, power_on=power_on, resize=resize_instance,
completed_callback=completed_callback)
def _start(self, instance, vm_ref=None, bad_volumes_callback=None):
def _start(self, instance, vm_ref=None, bad_volumes_callback=None,
start_pause=False):
"""Power on a VM instance."""
vm_ref = vm_ref or self._get_vm_opaque_ref(instance)
LOG.debug("Starting instance", instance=instance)
@@ -339,7 +341,7 @@ class VMOps(object):
self._session.call_xenapi('VM.start_on', vm_ref,
self._session.host_ref,
False, False)
start_pause, False)
# Allow higher-layers a chance to detach bad-volumes as well (in order
# to cleanup BDM entries and detach in Cinder)
@@ -548,14 +550,13 @@ class VMOps(object):
self._prepare_instance_filter(instance, network_info)
@step
def boot_instance_step(undo_mgr, vm_ref):
def start_paused_step(undo_mgr, vm_ref):
if power_on:
self._start(instance, vm_ref)
self._wait_for_instance_to_start(instance, vm_ref)
self._update_last_dom_id(vm_ref)
self._start(instance, vm_ref, start_pause=True)
@step
def configure_booted_instance_step(undo_mgr, vm_ref):
def boot_and_configure_instance_step(undo_mgr, vm_ref):
self._unpause_and_wait(vm_ref, instance, power_on)
if first_boot:
self._configure_new_instance_with_agent(instance, vm_ref,
injected_files, admin_password)
@@ -583,22 +584,67 @@ class VMOps(object):
attach_devices_step(undo_mgr, vm_ref, vdis, disk_image_type)
inject_instance_data_step(undo_mgr, vm_ref, vdis)
setup_network_step(undo_mgr, vm_ref)
if rescue:
attach_orig_disks_step(undo_mgr, vm_ref)
# if use neutron, prepare waiting event from neutron
# first_boot is True in new booted instance
# first_boot is False in migration and we don't waiting
# for neutron event regardless of whether or not it is
# migrated to another host, if unplug VIFs locally, the
# port status may not changed in neutron side and we
# cannot get the vif plug event from neturon
timeout = CONF.vif_plugging_timeout
events = self._get_neutron_events(network_info,
power_on, first_boot)
try:
with self._virtapi.wait_for_instance_event(
instance, events, deadline=timeout,
error_callback=self._neutron_failed_callback):
LOG.debug("wait for instance event:%s", events)
setup_network_step(undo_mgr, vm_ref)
if rescue:
attach_orig_disks_step(undo_mgr, vm_ref)
start_paused_step(undo_mgr, vm_ref)
except eventlet.timeout.Timeout:
self._handle_neutron_event_timeout(instance, undo_mgr)
boot_instance_step(undo_mgr, vm_ref)
configure_booted_instance_step(undo_mgr, vm_ref)
apply_security_group_filters_step(undo_mgr)
boot_and_configure_instance_step(undo_mgr, vm_ref)
if completed_callback:
completed_callback()
except Exception:
msg = _("Failed to spawn, rolling back")
undo_mgr.rollback_and_reraise(msg=msg, instance=instance)
def _handle_neutron_event_timeout(self, instance, undo_mgr):
# We didn't get callback from Neutron within given time
LOG.warn(_LW('Timeout waiting for vif plugging callback'),
instance=instance)
if CONF.vif_plugging_is_fatal:
raise exception.VirtualInterfaceCreateException()
def _unpause_and_wait(self, vm_ref, instance, power_on):
if power_on:
LOG.debug("Update instance when power on", instance=instance)
self._session.VM.unpause(vm_ref)
self._wait_for_instance_to_start(instance, vm_ref)
self._update_last_dom_id(vm_ref)
def _neutron_failed_callback(self, event_name, instance):
LOG.warn(_LW('Neutron Reported failure on event %(event)s'),
{'event': event_name}, instance=instance)
if CONF.vif_plugging_is_fatal:
raise exception.VirtualInterfaceCreateException()
def _get_neutron_events(self, network_info, power_on, first_boot):
# Only get network-vif-plugged events with VIF's status is not active.
# With VIF whose status is active, neutron may not notify such event.
timeout = CONF.vif_plugging_timeout
if (utils.is_neutron() and power_on and timeout and first_boot):
return [('network-vif-plugged', vif['id'])
for vif in network_info if vif.get('active', True) is False]
else:
return []
def _attach_orig_disks(self, instance, vm_ref):
orig_vm_ref = vm_utils.lookup(self._session, instance['name'])
orig_vdi_refs = self._find_vdi_refs(orig_vm_ref,