#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import time
from unittest import mock


from nova import exception
from nova.tests import fixtures
from nova.tests.functional.api import client
from nova.tests.functional.notification_sample_tests \
    import notification_sample_base
from nova.volume import cinder
from oslo_concurrency import processutils


class TestInstanceNotificationSampleWithMultipleCompute(
        notification_sample_base.NotificationSampleTestBase):

    def setUp(self):
        self.flags(compute_driver='fake.FakeLiveMigrateDriver')
        self.flags(bdms_in_notifications='True', group='notifications')
        self.flags(include_share_mapping='True', group='notifications')
        super(TestInstanceNotificationSampleWithMultipleCompute, self).setUp()
        self.neutron = fixtures.NeutronFixture(self)
        self.useFixture(self.neutron)
        self.cinder = fixtures.CinderFixture(self)
        self.useFixture(self.cinder)
        self.useFixture(fixtures.AllServicesCurrent())

    def test_multiple_compute_actions(self):
        # There are not going to be real network-vif-plugged events coming
        # so don't wait for them.
        self.flags(live_migration_wait_for_vif_plug=False, group='compute')
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._wait_for_notification('instance.create.end')
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
        # server will boot on the 'compute' host
        self.compute2 = self.start_service('compute', host='host2')

        actions = [
            (self._test_live_migration_rollback, 'ACTIVE'),
            (self._test_live_migration_abort, 'ACTIVE'),
            (self._test_live_migration_success, 'ACTIVE'),
            (self._test_evacuate_server, 'SHUTOFF'),
            (self._test_live_migration_force_complete, 'ACTIVE'),
        ]

        for action, expected_state in actions:
            self.notifier.reset()
            action(server)
            # Ensure that instance is in active state after an action
            self._wait_for_state_change(server, expected_state)

    @mock.patch('nova.compute.manager.ComputeManager.'
                '_live_migration_cleanup_flags', return_value=[True, False])
    @mock.patch('nova.compute.rpcapi.ComputeAPI.pre_live_migration',
                side_effect=exception.DestinationDiskExists(path='path'))
    def _test_live_migration_rollback(self, server, mock_migration,
                                      mock_flags):
        post = {
            'os-migrateLive': {
                'host': 'host2',
                'block_migration': True,
            }
        }
        self.admin_api.post_server_action(server['id'], post)
        self._wait_for_notification(
            'instance.live_migration_rollback_dest.end')

        # 0. scheduler.select_destinations.start
        # 1. scheduler.select_destinations.end
        # 2. instance.live_migration_rollback.start
        # 3. instance.live_migration_rollback.end
        # 4. instance.live_migration_rollback_dest.start
        # 5. instance.live_migration_rollback_dest.end
        self.assertEqual(6, len(self.notifier.versioned_notifications),
                         [x['event_type'] for x in
                          self.notifier.versioned_notifications])
        self._verify_notification(
            'instance-live_migration_rollback-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-live_migration_rollback-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])
        self._verify_notification(
            'instance-live_migration_rollback_dest-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[4])
        self._verify_notification(
            'instance-live_migration_rollback_dest-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])

    def _test_live_migration_success(self, server):
        post = {
            'os-migrateLive': {
                'host': 'host2',
                'block_migration': True,
            }
        }
        self.admin_api.post_server_action(server['id'], post)
        self._wait_for_notification('instance.live_migration_pre.end')
        # 0. scheduler.select_destinations.start
        # 1. scheduler.select_destinations.end
        # 2. instance.live_migration_pre.start
        # 3. instance.live_migration_pre.end
        self.assertEqual(4, len(self.notifier.versioned_notifications),
                         [x['event_type'] for x in
                          self.notifier.versioned_notifications])
        self._verify_notification(
            'instance-live_migration_pre-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-live_migration_pre-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])
        migrations = self.admin_api.get_active_migrations(server['id'])
        self.assertEqual(1, len(migrations))

        self._wait_for_notification('instance.live_migration_post.end')
        # 0. scheduler.select_destinations.start
        # 1. scheduler.select_destinations.end
        # 2. instance.live_migration_pre.start
        # 3. instance.live_migration_pre.end
        # 4. instance.live_migration_post.start
        # 5. instance.live_migration_post_dest.start
        # 6. instance.live_migration_post_dest.end
        # 7. instance.live_migration_post.end
        self.assertEqual(8, len(self.notifier.versioned_notifications),
                         [x['event_type'] for x in
                          self.notifier.versioned_notifications])
        self._verify_notification(
            'instance-live_migration_post-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[4])
        self._verify_notification(
            'instance-live_migration_post_dest-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])
        self._verify_notification(
            'instance-live_migration_post_dest-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[6])
        self._verify_notification(
            'instance-live_migration_post-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[7])

    def _test_live_migration_abort(self, server):
        post = {
            "os-migrateLive": {
                "host": "host2",
                "block_migration": False,
            }
        }

        self.admin_api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'MIGRATING')

        migrations = self._wait_and_get_migrations(server)

        self.admin_api.delete_migration(server['id'], migrations[0]['id'])
        self._wait_for_notification('instance.live_migration_abort.start')
        self._wait_for_state_change(server, 'ACTIVE')
        # NOTE(gibi): the instance.live_migration_rollback notification emitted
        # after the instance.live_migration_abort notification so we have to
        # wait for the rollback to ensure we can assert both notifications
        # below
        self._wait_for_notification('instance.live_migration_rollback.end')

        # 0. scheduler.select_destinations.start
        # 1. scheduler.select_destinations.end
        # 2. instance.live_migration_pre.start
        # 3. instance.live_migration_pre.end
        # 4. instance.live_migration_abort.start
        # 5. instance.live_migration_abort.end
        # 6. instance.live_migration_rollback.start
        # 7. instance.live_migration_rollback.end
        self.assertEqual(8, len(self.notifier.versioned_notifications),
                         [x['event_type'] for x in
                          self.notifier.versioned_notifications])
        self._verify_notification(
            'instance-live_migration_pre-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-live_migration_pre-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])
        self._verify_notification(
            'instance-live_migration_abort-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[4])
        self._verify_notification(
            'instance-live_migration_abort-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])
        self._verify_notification(
            'instance-live_migration_rollback-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[6])
        self._verify_notification(
            'instance-live_migration_rollback-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[7])

    def _test_evacuate_server(self, server):
        services = self.admin_api.get_services(host='host2',
                                               binary='nova-compute')
        service_id = services[0]['id']
        self.compute2.stop()
        self.admin_api.put_service(service_id, {'forced_down': True})

        post_args = {
            "host": "compute"
        }

        self._evacuate_server(
            server, extra_post_args=post_args, expected_host='compute')

        notifications = self._get_notifications('instance.evacuate')
        self.assertEqual(1, len(notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-evacuate',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=notifications[0])
        self.compute2.start()
        self._wait_for_migration_status(server, ['completed'])
        self.admin_api.put_service(service_id, {'forced_down': False})

    def _test_live_migration_force_complete(self, server):
        # In the scenario evacuate happened before which stopped the
        # server.
        self._start_server(server)
        self._wait_for_state_change(server, 'ACTIVE')
        self.notifier.reset()

        post = {
            'os-migrateLive': {
                'host': 'host2',
                'block_migration': True,
            }
        }
        self.admin_api.post_server_action(server['id'], post)

        self._wait_for_state_change(server, 'MIGRATING')

        migrations = self._wait_and_get_migrations(server)
        migration_id = migrations[0]['id']
        self.admin_api.force_complete_migration(server['id'], migration_id)

        # Note that we wait for instance.live_migration_force_complete.end but
        # by the time we check versioned notifications received we could have
        # entered ComputeManager._post_live_migration which could emit up to
        # four other notifications:
        # - instance.live_migration_post.start
        # - instance.live_migration_post_dest.start
        # - instance.live_migration_post_dest.end
        # - instance.live_migration_post.end
        # We are not concerned about those in this test so that's why we stop
        # once we get instance.live_migration_force_complete.end and assert
        # we got at least 6 notifications.
        self._wait_for_notification(
            'instance.live_migration_force_complete.end')

        # 0. scheduler.select_destinations.start
        # 1. scheduler.select_destinations.end
        # 2. instance.live_migration_pre.start
        # 3. instance.live_migration_pre.end
        # 4. instance.live_migration_force_complete.start
        # 5. instance.live_migration_force_complete.end
        self.assertGreaterEqual(len(self.notifier.versioned_notifications), 6,
                                self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-live_migration_force_complete-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[4])
        self._verify_notification(
            'instance-live_migration_force_complete-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])


class TestInstanceNotificationSample(
        notification_sample_base.NotificationSampleTestBase):

    def setUp(self):
        self.flags(bdms_in_notifications='True', group='notifications')
        self.flags(include_share_mapping='True', group='notifications')
        super(TestInstanceNotificationSample, self).setUp()
        self.neutron = fixtures.NeutronFixture(self)
        self.useFixture(self.neutron)
        self.cinder = fixtures.CinderFixture(self)
        self.useFixture(self.cinder)
        self.manila_fixture = self.useFixture(fixtures.ManilaFixture())

    def _wait_until_swap_volume(self, server, volume_id):
        for i in range(50):
            volume_attachments = self.api.get_server_volumes(server['id'])
            if len(volume_attachments) > 0:
                for volume_attachment in volume_attachments:
                    if volume_attachment['volumeId'] == volume_id:
                        return
            time.sleep(0.5)
        self.fail('Volume swap operation failed.')

    def test_instance_action(self):
        # A single test case is used to test most of the instance action
        # notifications to avoid booting up an instance for every action
        # separately.
        # Every instance action test function shall make sure that after the
        # function the instance is in active state and usable by other actions.
        # Therefore some action especially delete cannot be used here as
        # recovering from that action would mean to recreate the instance and
        # that would go against the whole purpose of this optimization

        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})

        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        actions = [
            self._test_power_off_on_server,
            self._test_restore_server,
            self._test_suspend_resume_server,
            self._test_pause_unpause_server,
            self._test_shelve_and_shelve_offload_server,
            self._test_unshelve_server,
            self._test_resize_and_revert_server,
            self._test_snapshot_server,
            self._test_reboot_server,
            self._test_reboot_server_error,
            self._test_trigger_crash_dump,
            self._test_volume_detach_attach_server,
            self._test_rescue_unrescue_server,
            self._test_soft_delete_server,
            self._test_attach_volume_error,
            self._test_interface_attach_and_detach,
            self._test_interface_attach_error,
            self._test_lock_unlock_instance,
            self._test_lock_unlock_instance_with_reason,
            self._test_share_attach_detach,
            self._test_share_attach_detach_error,
        ]

        for action in actions:
            self.notifier.reset()
            action(server)
            # Ensure that instance is in active state after an action
            self._wait_for_state_change(server, 'ACTIVE')

            # if the test step did not raised then we consider the step as
            # succeeded. We drop the logs to avoid causing subunit parser
            # errors due to logging too much at the end of the test case.
            self.stdlog.delete_stored_logs()

    def test_create_delete_server(self):
        fake_trusted_certs = ['cert-id-1', 'cert-id-2']
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}],
                          'tags': ['tag'],
                          'trusted_image_certificates': fake_trusted_certs})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
        self._delete_server(server)
        # NOTE(gibi): The wait_unit_deleted() call polls the REST API to see if
        # the instance is disappeared however the _delete_instance() in
        # compute/manager destroys the instance first then send the
        # instance.delete.end notification. So to avoid race condition the test
        # needs to wait for the notification as well here.
        self._wait_for_notification('instance.delete.end')
        self.assertEqual(9, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)

        # This list needs to be in order.
        expected_notifications = [
            'instance-create-start',
            'instance-create-end',
            'instance-update-tags-action',
            'instance-volume_attach-start',
            'instance-volume_attach-end',
            'instance-delete-start',
            'instance-shutdown-start',
            'instance-shutdown-end',
            'instance-delete-end'
        ]
        for idx, notification in enumerate(expected_notifications):
            self._verify_notification(
                notification,
                replacements={
                    'reservation_id': server['reservation_id'],
                    'uuid': server['id']},
                actual=self.notifier.versioned_notifications[idx])

    @mock.patch('nova.compute.manager.ComputeManager._build_resources')
    def test_create_server_error(self, mock_build):
        def _build_resources(*args, **kwargs):
            raise exception.FlavorDiskTooSmall()

        mock_build.side_effect = _build_resources

        fake_trusted_certs = ['cert-id-1', 'cert-id-2']
        server = self._boot_a_server(
            expected_status='ERROR',
            extra_params={'networks': [{'port': self.neutron.port_1['id']}],
                          'tags': ['tag'],
                          'trusted_image_certificates': fake_trusted_certs})

        # 0. scheduler.select_destinations.start
        # 1. scheduler.select_destinations.end
        # 2. instance-create-start
        # 3. instance-create-error
        self.assertEqual(4, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)

        tb = self.notifier.versioned_notifications[3]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn('raise exception.FlavorDiskTooSmall()', tb)

        self._verify_notification(
            'instance-create-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-create-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'fault.traceback': self.ANY},
            actual=self.notifier.versioned_notifications[3])

        self.notifier.reset()

        self._delete_server(server)

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-delete-start_not_scheduled',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-delete-end_not_scheduled',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def test_instance_exists_usage_audit(self):
        # TODO(xavvior): Should create a functional test for the
        # "instance_usage_audit" periodic task. We didn't find usable
        # solution for this problem, however we tried to test it in
        # several ways.
        pass

    def test_instance_exists(self):
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})

        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        self.notifier.reset()

        post = {
            'rebuild': {
                'imageRef': 'a2459075-d96c-40d5-893e-577ff92e721c',
                'metadata': {}
            }
        }
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, expected_status='REBUILD')
        self._wait_for_state_change(server, expected_status='ACTIVE')

        notifications = self._get_notifications('instance.exists')
        self._verify_notification(
            'instance-exists',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']
            },
            actual=notifications[0])

    def test_delete_server_while_compute_is_down(self):

        server = self._boot_a_server(
            expected_status='ACTIVE',
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        service_id = self.api.get_service_id('nova-compute')
        self.admin_api.put_service_force_down(service_id, True)
        self.notifier.reset()

        self._delete_server(server)

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-delete-start_compute_down',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-delete-end_compute_down',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

        self.admin_api.put_service_force_down(service_id, False)

    def _verify_instance_update_steps(self, steps, notifications,
                                      initial=None):
        replacements = {}
        if initial:
            replacements = initial
        for i, step in enumerate(steps):
            replacements.update(step)
            self._verify_notification(
                'instance-update',
                replacements=replacements,
                actual=notifications[i])
        return replacements

    def test_create_delete_server_with_instance_update(self):
        # This makes server network creation synchronous which is necessary
        # for notification samples that expect instance.info_cache.network_info
        # to be set.
        self.useFixture(fixtures.SpawnIsSynchronousFixture())
        self.flags(notify_on_state_change='vm_and_task_state',
                   group='notifications')

        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        instance_updates = self._wait_for_notifications('instance.update', 8)

        # The first notification comes from the nova-conductor, the
        # eighth notification comes from nova-api the
        # rest is from the nova-compute. To keep the test simpler
        # assert this fact and then modify the publisher_id of the
        # first and eighth notification to match the template
        self.assertEqual('nova-conductor:fake-mini',
                         instance_updates[0]['publisher_id'])
        self.assertEqual('nova-api:fake-mini',
                         instance_updates[7]['publisher_id'])
        instance_updates[0]['publisher_id'] = 'nova-compute:fake-mini'
        instance_updates[7]['publisher_id'] = 'nova-compute:fake-mini'

        create_steps = [
            # nothing -> scheduling
            {'reservation_id': server['reservation_id'],
             'uuid': server['id'],
             'host': None,
             'node': None,
             'state_update.new_task_state': 'scheduling',
             'state_update.old_task_state': 'scheduling',
             'state_update.state': 'building',
             'state_update.old_state': 'building',
             'state': 'building'},

            # scheduling -> building
            {
             'state_update.new_task_state': None,
             'state_update.old_task_state': 'scheduling',
             'task_state': None},

            # scheduled
            {'host': 'compute',
             'node': 'fake-mini',
             'state_update.old_task_state': None,
             'updated_at': '2012-10-29T13:42:11Z'},

            # building -> networking
            {'state_update.new_task_state': 'networking',
             'state_update.old_task_state': 'networking',
             'task_state': 'networking'},

            # networking -> block_device_mapping
            {'state_update.new_task_state': 'block_device_mapping',
             'state_update.old_task_state': 'networking',
             'task_state': 'block_device_mapping',
             'ip_addresses': [{
                 "nova_object.name": "IpPayload",
                 "nova_object.namespace": "nova",
                 "nova_object.version": "1.0",
                 "nova_object.data": {
                     "mac": "fa:16:3e:4c:2c:30",
                     "address": "192.168.1.3",
                     "port_uuid": "ce531f90-199f-48c0-816c-13e38010b442",
                     "meta": {},
                     "version": 4,
                     "label": "private",
                     "device_name": "tapce531f90-19"
                 }}]
            },

            # block_device_mapping -> spawning
            {'state_update.new_task_state': 'spawning',
             'state_update.old_task_state': 'block_device_mapping',
             'task_state': 'spawning',
             },

            # spawning -> active
            {'state_update.new_task_state': None,
             'state_update.old_task_state': 'spawning',
             'state_update.state': 'active',
             'launched_at': '2012-10-29T13:42:11Z',
             'state': 'active',
             'task_state': None,
             'power_state': 'running'},

            # tag added
            {'state_update.old_task_state': None,
             'state_update.old_state': 'active',
             'tags': ['tag1']},
        ]

        replacements = self._verify_instance_update_steps(
                create_steps, instance_updates)

        self.notifier.reset()

        self._delete_server(server)

        instance_updates = self._get_notifications('instance.update')
        self.assertEqual(2, len(instance_updates),
                         self.notifier.versioned_notifications)

        delete_steps = [
            # active -> deleting
            {'state_update.new_task_state': 'deleting',
             'state_update.old_task_state': 'deleting',
             'state_update.old_state': 'active',
             'state': 'active',
             'task_state': 'deleting',
             'tags': ["tag1"],
             'block_devices': [{
                "nova_object.data": {
                    "boot_index": None,
                    "delete_on_termination": False,
                    "device_name": "/dev/sdb",
                    "tag": None,
                    "volume_id": "a07f71dc-8151-4e7d-a0cc-cd24a3f11113"
                },
                "nova_object.name": "BlockDevicePayload",
                "nova_object.namespace": "nova",
                "nova_object.version": "1.0"
              }]
            },

            # deleting -> deleted
            {'state_update.new_task_state': None,
             'state_update.old_task_state': 'deleting',
             'state_update.old_state': 'active',
             'state_update.state': 'deleted',
             'state': 'deleted',
             'task_state': None,
             'terminated_at': '2012-10-29T13:42:11Z',
             'ip_addresses': [],
             'power_state': 'pending',
             'tags': ["tag1"],
             'block_devices': [{
                "nova_object.data": {
                    "boot_index": None,
                    "delete_on_termination": False,
                    "device_name": "/dev/sdb",
                    "tag": None,
                    "volume_id": "a07f71dc-8151-4e7d-a0cc-cd24a3f11113"
                },
                "nova_object.name": "BlockDevicePayload",
                "nova_object.namespace": "nova",
                "nova_object.version": "1.0"
              }]
            },
        ]

        self._verify_instance_update_steps(delete_steps, instance_updates,
                                           initial=replacements)

    def _test_power_off_on_server(self, server):
        self.api.post_server_action(server['id'], {'os-stop': {}})
        self._wait_for_state_change(server, expected_status='SHUTOFF')
        self.api.post_server_action(server['id'], {'os-start': {}})
        self._wait_for_state_change(server, expected_status='ACTIVE')

        self.assertEqual(4, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-power_off-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-power_off-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

        self._verify_notification(
            'instance-power_on-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-power_on-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])

    def _test_shelve_and_shelve_offload_server(self, server):
        self.flags(shelved_offload_time=-1)
        self.api.post_server_action(server['id'], {'shelve': {}})
        self._wait_for_state_change(server, expected_status='SHELVED')

        self.assertEqual(3, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-shelve-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])
        self._verify_notification(
            'instance-shelve-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])

        self.notifier.reset()
        self.api.post_server_action(server['id'], {'shelveOffload': {}})
        # we need to wait for the instance.host to become None as well before
        # we can unshelve to make sure that the unshelve.start notification
        # payload is stable as the compute manager first sets the instance
        # state then a bit later sets the instance.host to None.
        self._wait_for_server_parameter(server,
                                        {'status': 'SHELVED_OFFLOADED',
                                         'OS-EXT-SRV-ATTR:host': None})

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-shelve_offload-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-shelve_offload-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

        self.api.post_server_action(server['id'], {'unshelve': None})
        self._wait_for_state_change(server, 'ACTIVE')
        self._wait_for_notification('instance.unshelve.end')

    def _test_unshelve_server(self, server):
        # setting the shelved_offload_time to 0 should set the
        # instance status to 'SHELVED_OFFLOADED'
        self.flags(shelved_offload_time = 0)
        self.api.post_server_action(server['id'], {'shelve': {}})
        # we need to wait for the instance.host to become None as well before
        # we can unshelve to make sure that the unshelve.start notification
        # payload is stable as the compute manager first sets the instance
        # state then a bit later sets the instance.host to None.
        self._wait_for_server_parameter(server,
                                        {'status': 'SHELVED_OFFLOADED',
                                         'OS-EXT-SRV-ATTR:host': None})

        post = {'unshelve': None}
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'ACTIVE')
        self._wait_for_notification('instance.unshelve.end')
        self.assertEqual(9, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-unshelve-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[7])
        self._verify_notification(
            'instance-unshelve-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[8])

    def _test_suspend_resume_server(self, server):
        post = {'suspend': {}}
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'SUSPENDED')

        post = {'resume': None}
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'ACTIVE')

        # Four versioned notification are generated.
        # 0. instance-suspend-start
        # 1. instance-suspend-end
        # 2. instance-resume-start
        # 3. instance-resume-end
        self.assertEqual(4, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-suspend-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-suspend-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

        self._verify_notification(
            'instance-resume-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-resume-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])

        self.flags(reclaim_instance_interval=0)

    def _test_pause_unpause_server(self, server):
        self.api.post_server_action(server['id'], {'pause': {}})
        self._wait_for_state_change(server, 'PAUSED')

        self.api.post_server_action(server['id'], {'unpause': {}})
        self._wait_for_state_change(server, 'ACTIVE')

        # Four versioned notifications are generated
        # 0. instance-pause-start
        # 1. instance-pause-end
        # 2. instance-unpause-start
        # 3. instance-unpause-end
        self.assertEqual(4, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-pause-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-pause-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])
        self._verify_notification(
            'instance-unpause-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-unpause-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])

    def _build_destination_payload(self):
        cell1 = self.cell_mappings.get('cell1')
        return {
            'nova_object.version': '1.0',
            'nova_object.namespace': 'nova',
            'nova_object.name': 'DestinationPayload',
            'nova_object.data': {
                'aggregates': None,
                'cell': {
                    'nova_object.version': '2.0',
                    'nova_object.namespace': 'nova',
                    'nova_object.name': 'CellMappingPayload',
                    'nova_object.data': {
                        'disabled': False,
                        'name': u'cell1',
                        'uuid': cell1.uuid
                    }
                }
            }
        }

    def _test_resize_and_revert_server(self, server):
        self.flags(allow_resize_to_same_host=True)
        other_flavor_body = {
            'flavor': {
                'name': 'other_flavor',
                'ram': 256,
                'vcpus': 1,
                'disk': 1,
                'id': 'd5a8bb54-365a-45ae-abdb-38d249df7845'
            }
        }
        other_flavor_id = self.api.post_flavor(other_flavor_body)['id']
        extra_specs = {
            "extra_specs": {
                "hw:watchdog_action": "reset"}}
        self.admin_api.post_extra_spec(other_flavor_id, extra_specs)

        # Ignore the create flavor notification
        self.notifier.reset()

        post = {
            'resize': {
                'flavorRef': other_flavor_id
            }
        }
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'VERIFY_RESIZE')

        self._pop_and_verify_dest_select_notification(server['id'],
            replacements={
                'ignore_hosts': [],
                'flavor.memory_mb': other_flavor_body['flavor']['ram'],
                'flavor.name': other_flavor_body['flavor']['name'],
                'flavor.flavorid': other_flavor_id,
                'flavor.extra_specs': extra_specs['extra_specs'],
                'requested_destination': self._build_destination_payload()})

        self.assertEqual(7, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        # ignore instance.exists
        self.notifier.versioned_notifications.pop(0)

        # This list needs to be in order.
        expected_notifications = [
            'instance-resize_prep-start',
            'instance-resize_prep-end',
            'instance-resize-start',
            'instance-resize-end',
            'instance-resize_finish-start',
            'instance-resize_finish-end'
        ]
        for idx, notification in enumerate(expected_notifications):
            self._verify_notification(
                notification,
                replacements={
                    'reservation_id': server['reservation_id'],
                    'uuid': server['id']},
                actual=self.notifier.versioned_notifications[idx])

        self.notifier.reset()
        # the following is the revert server request
        post = {'revertResize': None}
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'ACTIVE')

        self.assertEqual(3, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        # ignore instance.exists
        self.notifier.versioned_notifications.pop(0)
        self._verify_notification(
            'instance-resize_revert-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-resize_revert-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    @mock.patch('nova.compute.manager.ComputeManager._prep_resize')
    def test_resize_server_error_but_reschedule_was_success(
            self, mock_prep_resize):
        """Test it, when the prep_resize method raise an exception,
        but the reschedule_resize_or_reraise was successful and
        scheduled the resize. In this case we get a notification
        about the exception, which caused the prep_resize error.
        """
        def _build_resources(*args, **kwargs):
            raise exception.FlavorDiskTooSmall()
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self.flags(allow_resize_to_same_host=True)
        other_flavor_body = {
            'flavor': {
                'name': 'other_flavor_error',
                'ram': 512,
                'vcpus': 1,
                'disk': 1,
                'id': 'a22d5517-147c-4147-a0d1-e698df5cd4e9'
            }
        }
        other_flavor_id = self.api.post_flavor(other_flavor_body)['id']

        post = {
            'resize': {
                'flavorRef': other_flavor_id
            }
        }
        self.notifier.reset()
        mock_prep_resize.side_effect = _build_resources
        # NOTE(gibi): the first resize_instance call (from the API) should be
        # unaffected so that we can reach _prep_resize at all. But the
        # subsequent resize_instance call (from _reschedule_resize_or_reraise)
        # needs to be mocked as there is no alternative host to resize to.
        patcher = mock.patch.object(self.compute.manager.compute_task_api,
                                    'resize_instance')
        self.addCleanup(patcher.stop)
        patcher.start()
        self.api.post_server_action(server['id'], post)
        self._wait_for_notification('instance.resize.error')
        self._pop_and_verify_dest_select_notification(server['id'],
            replacements={
                'ignore_hosts': [],
                'flavor.name': other_flavor_body['flavor']['name'],
                'flavor.flavorid': other_flavor_id,
                'flavor.extra_specs': {},
                'requested_destination': self._build_destination_payload()})
        # 0: instance-exists
        # 1: instance-resize_prep-start
        # 2: instance-resize-error
        # 3: instance-resize_prep-end
        self.assertLessEqual(2, len(self.notifier.versioned_notifications),
                             'Unexpected number of notifications: %s' %
                             self.notifier.versioned_notifications)
        # Note(gibi): There is also an instance.exists notification emitted
        # during the rescheduling

        tb = self.notifier.versioned_notifications[2]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn("raise exception.FlavorDiskTooSmall()", tb)

        self._verify_notification('instance-resize-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'fault.traceback': self.ANY
            },
            actual=self.notifier.versioned_notifications[2])

    @mock.patch('nova.compute.manager.ComputeManager._prep_resize')
    def test_resize_server_error_and_reschedule_was_failed(
            self, mock_prep_resize):
        """Test it, when the prep_resize method raise an exception,
        after trying again with the reschedule_resize_or_reraise method
        call, but the rescheduled also was unsuccessful. In this
        case called the exception block.
        In the exception block send a notification about error.
        At end called raising an exception based on *exc_info,
        which not send another error.
        """
        def _build_resources(*args, **kwargs):
            raise exception.FlavorDiskTooSmall()

        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self.flags(allow_resize_to_same_host=True)
        other_flavor_body = {
            'flavor': {
                'name': 'other_flavor_error',
                'ram': 512,
                'vcpus': 1,
                'disk': 1,
                'id': 'a22d5517-147c-4147-a0d1-e698df5cd4e9'
            }
        }
        other_flavor_id = self.api.post_flavor(other_flavor_body)['id']

        post = {
            'resize': {
                'flavorRef': other_flavor_id
            }
        }
        self.notifier.reset()
        mock_prep_resize.side_effect = _build_resources
        # NOTE(gibi): the first resize_instance call (from the API) should be
        # unaffected so that we can reach _prep_resize at all. But the
        # subsequent resize_instance call (from _reschedule_resize_or_reraise)
        # needs to fail. It isn't realistic that resize_instance would raise
        # FlavorDiskTooSmall, but it's needed for the notification sample
        # to work.
        patcher = mock.patch.object(self.compute.manager.compute_task_api,
                                    'resize_instance',
                                    side_effect=_build_resources)
        self.addCleanup(patcher.stop)
        patcher.start()
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, expected_status='ERROR')
        self._wait_for_notification('compute.exception')
        # There should be the following notifications after scheduler's
        # select_destination notifications:
        # 0: instance-exists
        # 1: instance-resize_prep-start
        # 2: instance-resize-error
        # 3: instance-resize_prep-end
        # 4: compute.exception
        #    (via the wrap_exception decorator on
        #    the ComputeManager.prep_resize method.)
        self._pop_and_verify_dest_select_notification(server['id'],
            replacements={
                'ignore_hosts': [],
                'flavor.name': other_flavor_body['flavor']['name'],
                'flavor.flavorid': other_flavor_id,
                'flavor.extra_specs': {},
                'requested_destination': self._build_destination_payload()})
        self.assertEqual(5, len(self.notifier.versioned_notifications),
                         'Unexpected number of notifications: %s' %
                         self.notifier.versioned_notifications)

        tb = self.notifier.versioned_notifications[2]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn("raise exception.FlavorDiskTooSmall()", tb)

        self._verify_notification('instance-resize-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'fault.traceback': self.ANY
            },
            actual=self.notifier.versioned_notifications[2])

    def _test_snapshot_server(self, server):
        post = {'createImage': {'name': 'test-snap'}}
        response = self.api.post_server_action(server['id'], post)
        self._wait_for_notification('instance.snapshot.end')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-snapshot-start',
            replacements={
                'snapshot_image_id': response['image_id'],
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
                    actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-snapshot-end',
            replacements={
                'snapshot_image_id': response['image_id'],
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def test_rebuild_server(self):
        # NOTE(gabor_antal): Rebuild changes the image used by the instance,
        # therefore the actions tested in test_instance_action had to be in
        # specific order. To avoid this problem, rebuild was moved from
        # test_instance_action to its own method.

        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        self.notifier.reset()

        image_ref = 'a2459075-d96c-40d5-893e-577ff92e721c'
        post = {
            'rebuild': {
                'imageRef': image_ref,
                'metadata': {}
            }
        }
        self.api.post_server_action(server['id'], post)
        # Before going back to ACTIVE state
        # server state need to be changed to REBUILD state
        self._wait_for_state_change(server, expected_status='REBUILD')
        self._wait_for_state_change(server, expected_status='ACTIVE')

        self._pop_and_verify_dest_select_notification(server['id'],
            replacements={
                'image.container_format': 'ami',
                'image.disk_format': 'ami',
                'image.id': image_ref,
                'image.properties': {
                    'nova_object.data': {},
                    'nova_object.name': 'ImageMetaPropsPayload',
                    'nova_object.namespace': 'nova',
                    'nova_object.version': '1.18',
                },
                'image.size': 58145823,
                'image.tags': [],
                'scheduler_hints': {'_nova_check_type': ['rebuild']},
                'force_hosts': 'compute',
                'force_nodes': 'fake-mini',
                'requested_destination': self._build_destination_payload()})

        # 0. instance.rebuild_scheduled
        # 1. instance.exists
        # 2. instance.rebuild.start
        # 3. instance.detach.start
        # 4. instance.detach.end
        # 5. instance.rebuild.end
        # The compute/manager will detach every volume during rebuild
        self.assertEqual(6, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-rebuild_scheduled',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'trusted_image_certificates': None},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-rebuild-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'trusted_image_certificates': None},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-volume_detach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'task_state': 'rebuilding',
                'architecture': None,
                'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])
        self._verify_notification(
            'instance-volume_detach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'task_state': 'rebuilding',
                'architecture': None,
                'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[4])
        self._verify_notification(
            'instance-rebuild-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'trusted_image_certificates': None},
            actual=self.notifier.versioned_notifications[5])

    def test_rebuild_server_with_trusted_cert(self):
        # NOTE(gabor_antal): Rebuild changes the image used by the instance,
        # therefore the actions tested in test_instance_action had to be in
        # specific order. To avoid this problem, rebuild was moved from
        # test_instance_action to its own method.

        create_trusted_certs = ['cert-id-1', 'cert-id-2']
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}],
                          'trusted_image_certificates': create_trusted_certs})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        self.notifier.reset()

        image_ref = 'a2459075-d96c-40d5-893e-577ff92e721c'
        rebuild_trusted_certs = ['rebuild-cert-id-1', 'rebuild-cert-id-2']
        post = {
            'rebuild': {
                'imageRef': image_ref,
                'metadata': {},
                'trusted_image_certificates': rebuild_trusted_certs,
            }
        }
        self.api.post_server_action(server['id'], post)
        # Before going back to ACTIVE state
        # server state need to be changed to REBUILD state
        self._wait_for_state_change(server, expected_status='REBUILD')
        self._wait_for_state_change(server, expected_status='ACTIVE')

        self._pop_and_verify_dest_select_notification(server['id'],
            replacements={
                'image.container_format': 'ami',
                'image.disk_format': 'ami',
                'image.id': image_ref,
                'image.properties': {
                    'nova_object.data': {},
                    'nova_object.name': 'ImageMetaPropsPayload',
                    'nova_object.namespace': 'nova',
                    'nova_object.version': '1.18',
                },
                'image.size': 58145823,
                'image.tags': [],
                'scheduler_hints': {'_nova_check_type': ['rebuild']},
                'force_hosts': 'compute',
                'force_nodes': 'fake-mini',
                'requested_destination': self._build_destination_payload()})

        # 0. instance.rebuild_scheduled
        # 1. instance.exists
        # 2. instance.rebuild.start
        # 3. instance.detach.start
        # 4. instance.detach.end
        # 5. instance.rebuild.end
        # The compute/manager will detach every volume during rebuild
        self.assertEqual(6, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-rebuild_scheduled',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-rebuild-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self._verify_notification(
            'instance-volume_detach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'task_state': 'rebuilding',
                'architecture': None,
                'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[3])
        self._verify_notification(
            'instance-volume_detach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'task_state': 'rebuilding',
                'architecture': None,
                'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[4])
        self._verify_notification(
            'instance-rebuild-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])

    @mock.patch('nova.compute.manager.ComputeManager.'
                '_do_rebuild_instance_with_claim')
    def test_rebuild_server_exc(self, mock_rebuild):
        def _virtual_interface_create_failed(*args, **kwargs):
            # A real error that could come out of driver.spawn() during rebuild
            raise exception.VirtualInterfaceCreateException()

        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        self.notifier.reset()

        post = {
            'rebuild': {
                'imageRef': 'a2459075-d96c-40d5-893e-577ff92e721c',
                'metadata': {}
            }
        }
        self.api.post_server_action(server['id'], post)
        mock_rebuild.side_effect = _virtual_interface_create_failed
        self._wait_for_state_change(server, expected_status='ERROR')
        notification = self._get_notifications('instance.rebuild.error')
        self.assertEqual(1, len(notification),
                         self.notifier.versioned_notifications)

        tb = notification[0]['payload']['nova_object.data']['fault'][
                'nova_object.data']['traceback']
        self.assertIn('raise exception.VirtualInterfaceCreateException()', tb)

        self._verify_notification(
            'instance-rebuild-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'trusted_image_certificates': None,
                'fault.traceback': self.ANY},
            actual=notification[0])

    def _test_restore_server(self, server):
        self.flags(reclaim_instance_interval=30)
        self.api.delete_server(server['id'])
        self._wait_for_state_change(server, 'SOFT_DELETED')
        # we don't want to test soft_delete here
        self.notifier.reset()
        self.api.post_server_action(server['id'], {'restore': {}})
        self._wait_for_state_change(server, 'ACTIVE')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-restore-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-restore-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def _test_reboot_server(self, server):
        post = {'reboot': {'type': 'HARD'}}
        self.api.post_server_action(server['id'], post)
        self._wait_for_notification('instance.reboot.start')
        self._wait_for_notification('instance.reboot.end')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-reboot-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-reboot-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    @mock.patch('nova.virt.fake.SmallFakeDriver.reboot')
    def _test_reboot_server_error(self, server, mock_reboot):
        def _hard_reboot(*args, **kwargs):
            raise exception.UnsupportedVirtType(virt="FakeVirt")
        mock_reboot.side_effect = _hard_reboot
        post = {'reboot': {'type': 'HARD'}}
        self.api.post_server_action(server['id'], post)
        self._wait_for_notification('instance.reboot.start')
        self._wait_for_notification('instance.reboot.error')
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)

        tb = self.notifier.versioned_notifications[1]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn("raise exception.UnsupportedVirtType", tb)

        self._verify_notification(
            'instance-reboot-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-reboot-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'fault.traceback': self.ANY},
            actual=self.notifier.versioned_notifications[1])

    def _detach_volume_from_server(self, server, volume_id):
        self.api.delete_server_volume(server['id'], volume_id)
        self._wait_for_notification('instance.volume_detach.end')

    def _volume_swap_server(self, server, attachment_id, volume_id):
        self.api.put_server_volume(server['id'], attachment_id, volume_id)

    def test_volume_swap_server(self):
        server = self._boot_a_server(
            extra_params={'networks':
                          [{'port': self.neutron.port_1['id']}]})

        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
        self.cinder.swap_volume_instance_uuid = server['id']

        self._volume_swap_server(server, self.cinder.SWAP_OLD_VOL,
                                 self.cinder.SWAP_NEW_VOL)
        self._wait_until_swap_volume(server, self.cinder.SWAP_NEW_VOL)
        # NOTE(gibi): the new volume id can appear on the API earlier than the
        # volume_swap.end notification emitted. So to make the test stable
        # we have to wait for the volume_swap.end notification directly.
        self._wait_for_notification('instance.volume_swap.end')

        self.assertEqual(7, len(self.notifier.versioned_notifications),
                         'Unexpected number of versioned notifications. '
                         'Got: %s' % self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-volume_swap-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])
        self._verify_notification(
            'instance-volume_swap-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[6])

    def _do_setup_server_and_error_flag(self):
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._attach_volume_to_server(server, self.cinder.SWAP_ERR_OLD_VOL)

        self.cinder.attachment_error_id = self.cinder.SWAP_ERR_ATTACH_ID

        return server

    def test_volume_swap_server_with_error(self):
        server = self._do_setup_server_and_error_flag()

        # This is calling swap volume but we are emulating cinder
        # volume migrate in the fixture to allow this.
        # i.e. this is simulating the workflow where you move a volume
        # between cinder backend using a temp volume that cinder internally
        # cleans up at the end of the migration.
        self._volume_swap_server(
            server, self.cinder.SWAP_ERR_OLD_VOL,
            self.cinder.SWAP_ERR_NEW_VOL)
        self._wait_for_notification('compute.exception')

        # Eight versioned notifications are generated.
        # 0. instance-create-start
        # 1. instance-create-end
        # 2. instance-update
        # 3. instance-volume_attach-start
        # 4. instance-volume_attach-end
        # 5. instance-volume_swap-start
        # 6. instance-volume_swap-error
        # 7. compute.exception
        self.assertLessEqual(7, len(self.notifier.versioned_notifications),
                             'Unexpected number of versioned notifications. '
                             'Got: %s' % self.notifier.versioned_notifications)
        # the block device mapping is using SWAP_ERR_OLD_VOL because this is
        # the cinder volume migrate workflow.
        block_devices = [{
            "nova_object.data": {
                "boot_index": None,
                "delete_on_termination": False,
                "device_name": "/dev/sdb",
                "tag": None,
                "volume_id": self.cinder.SWAP_ERR_OLD_VOL
            },
            "nova_object.name": "BlockDevicePayload",
            "nova_object.namespace": "nova",
            "nova_object.version": "1.0"
        }]
        self._verify_notification(
            'instance-volume_swap-start',
            replacements={
                'new_volume_id': self.cinder.SWAP_ERR_NEW_VOL,
                'old_volume_id': self.cinder.SWAP_ERR_OLD_VOL,
                'block_devices': block_devices,
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[5])

        tb1 = self.notifier.versioned_notifications[6]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn("_swap_volume", tb1)
        tb2 = self.notifier.versioned_notifications[7]['payload'][
            'nova_object.data']['traceback']
        self.assertIn("_swap_volume", tb2)

        self._verify_notification(
            'instance-volume_swap-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'block_devices': block_devices,
                'uuid': server['id'],
                'fault.traceback': self.ANY},
            actual=self.notifier.versioned_notifications[6])

    def test_resize_confirm_server(self):
        server = self._boot_a_server(
            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
        self.admin_api.post_extra_spec(
            '2', {"extra_specs": {"hw:watchdog_action": "disabled"}})
        self.flags(allow_resize_to_same_host=True)
        post = {'resize': {'flavorRef': '2'}}
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'VERIFY_RESIZE')
        self.notifier.reset()

        post = {'confirmResize': None}
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'ACTIVE')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-resize_confirm-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-resize_confirm-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def _test_trigger_crash_dump(self, server):
        post = {'trigger_crash_dump': None}
        self.api.post_server_action(server['id'], post)

        self._wait_for_notification('instance.trigger_crash_dump.end')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-trigger_crash_dump-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-trigger_crash_dump-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def _test_volume_detach_attach_server(self, server):
        self._detach_volume_from_server(server, self.cinder.SWAP_OLD_VOL)

        # 0. volume_detach-start
        # 1. volume_detach-end
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-volume_detach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-volume_detach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

        self.notifier.reset()
        self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)

        # 0. volume_attach-start
        # 1. volume_attach-end
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-volume_attach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-volume_attach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def _test_share_attach_detach(self, server):

        expected_shares = [
            {
                "nova_object.name": "SharePayload",
                "nova_object.namespace": "nova",
                "nova_object.version": "1.0",
                "nova_object.data": {
                    "share_mapping_uuid": (
                        "f7c1726d-7622-42b3-8b2c-4473239d60d1"),
                    "share_id": "e8debdc0-447a-4376-a10a-4cd9122d7986",
                    "status": "attaching",
                    "tag": "e8debdc0-447a-4376-a10a-4cd9122d7986",
                    # 'export_location': '10.0.0.50:/mnt/foo',
                },
            }
        ]

        self.api.post_server_action(server['id'], {'os-stop': {}})
        self._wait_for_state_change(server, expected_status='SHUTOFF')
        self.notifier.reset()

        self._attach_share(server, "e8debdc0-447a-4376-a10a-4cd9122d7986")
        share_info = self._get_share(
            server, "e8debdc0-447a-4376-a10a-4cd9122d7986", admin=True
        )
        expected_shares[0]["nova_object.data"][
            "share_mapping_uuid"
        ] = share_info["uuid"]

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-share_attach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[0])
        expected_shares[0]['nova_object.data']['status'] = 'inactive'
        self._verify_notification(
            'instance-share_attach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[1])

        # Start server
        self.notifier.reset()
        self.api.post_server_action(server['id'], {'os-start': {}})
        self._wait_for_state_change(server, expected_status='ACTIVE')
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-power_on_share-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares,
            },
            actual=self.notifier.versioned_notifications[0])
        expected_shares[0]['nova_object.data']['status'] = 'active'
        self._verify_notification(
            'instance-power_on_share-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'active',
                'power_state': 'running',
                'shares': expected_shares,
            },
            actual=self.notifier.versioned_notifications[1])

        # Shutdown server
        self.notifier.reset()
        self.api.post_server_action(server['id'], {'os-stop': {}})
        self._wait_for_state_change(server, expected_status='SHUTOFF')
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-power_off-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'active',
                'power_state': 'running',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[0])
        expected_shares[0]['nova_object.data']['status'] = 'inactive'
        self._verify_notification(
            'instance-power_off-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[1])

        self.notifier.reset()

        # Detach share
        self._detach_share(server, 'e8debdc0-447a-4376-a10a-4cd9122d7986')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        expected_shares[0]['nova_object.data']['status'] = 'detaching'
        self._verify_notification(
            'instance-share_detach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-share_detach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
            },
            actual=self.notifier.versioned_notifications[1])

        # Start server
        self.notifier.reset()
        self.api.post_server_action(server['id'], {'os-start': {}})
        self._wait_for_state_change(server, expected_status='ACTIVE')
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-power_on-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
            },
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-power_on-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'active',
                'power_state': 'running',
            },
            actual=self.notifier.versioned_notifications[1])

    def _test_share_attach_detach_error(self, server):

        expected_shares = [
            {
                "nova_object.name": "SharePayload",
                "nova_object.namespace": "nova",
                "nova_object.version": "1.0",
                "nova_object.data": {
                    "share_mapping_uuid": (
                        "f7c1726d-7622-42b3-8b2c-4473239d60d1"),
                    "share_id": "e8debdc0-447a-4376-a10a-4cd9122d7986",
                    "status": "attaching",
                    "tag": "e8debdc0-447a-4376-a10a-4cd9122d7986",
                    # 'export_location': '10.0.0.50:/mnt/foo',
                },
            }
        ]

        self.api.post_server_action(server['id'], {'os-stop': {}})
        self._wait_for_state_change(server, expected_status='SHUTOFF')
        self.notifier.reset()

        with mock.patch(
            'nova.virt.fake.FakeDriver.mount_share',
            side_effect=exception.ShareMountError(
                share_id='8db0037b-e98f-4bde-ae71-f96a077c19a4',
                server_id=server['id'],
                reason=processutils.ProcessExecutionError(
                    stdout='This is stdout',
                    stderr='This is stderror',
                    exit_code=1,
                    cmd="mount"
                )
            )
        ):

            # Simulate an error attaching the share.
            original_side_effect = self.manila_fixture.mock_allow.side_effect
            self.manila_fixture.mock_allow.side_effect = (
                exception.ShareAccessGrantError(
                    share_id="8db0037b-e98f-4bde-ae71-f96a077c19a4",
                    reason="Connection timed out"
                )
            )

            self._attach_share_with_error(
                server, "e8debdc0-447a-4376-a10a-4cd9122d7986")

            share_info = self._get_share(
                server, "e8debdc0-447a-4376-a10a-4cd9122d7986", admin=True
            )
            expected_shares[0]["nova_object.data"][
                "share_mapping_uuid"
            ] = share_info["uuid"]

            # 0: instance-share_attach-start
            # 1: instance-share_attach-error
            # 2: compute.exception
            self.assertEqual(3, len(self.notifier.versioned_notifications),
                             self.notifier.versioned_notifications)
            self._verify_notification(
                'instance-share_attach-start',
                replacements={
                    'reservation_id': server['reservation_id'],
                    'uuid': server['id'],
                    'state': 'stopped',
                    'power_state': 'shutdown',
                    'shares': expected_shares,
                },
                actual=self.notifier.versioned_notifications[0])
            expected_shares[0]["nova_object.data"]["status"] = "error"
            self._verify_notification(
                'instance-share_attach-error',
                replacements={
                    'reservation_id': server['reservation_id'],
                    'uuid': server['id'],
                    'state': 'stopped',
                    'power_state': 'shutdown',
                    'shares': expected_shares,
                    'fault.traceback': self.ANY
                },
                actual=self.notifier.versioned_notifications[1])

            self._wait_for_notification('compute.exception')

            # Restart server
            self.notifier.reset()
            self.api.post_server_action(server['id'], {'os-start': {}})
            self._wait_for_state_change(server, expected_status='ERROR')

            self.assertTrue(
                any(
                    [
                        item
                        for item in self.notifier.versioned_notifications
                        if item["event_type"] == "compute.exception"
                    ]
                )
            )

            # Detach share despite the share error
            self.notifier.reset()
            self._detach_share(
                server, 'e8debdc0-447a-4376-a10a-4cd9122d7986')

            expected_shares[0]["nova_object.data"]["status"] = "detaching"
            self.assertEqual(2, len(self.notifier.versioned_notifications),
                             self.notifier.versioned_notifications)
            self._verify_notification(
                'instance-share_detach-start',
                replacements={
                    'reservation_id': server['reservation_id'],
                    'uuid': server['id'],
                    'state': 'error',
                    'shares': expected_shares,
                    'power_state': 'shutdown'},
                actual=self.notifier.versioned_notifications[0])
            expected_shares[0]['nova_object.data']['status'] = 'detaching'
            self._verify_notification(
                'instance-share_detach-end',
                replacements={
                    'reservation_id': server['reservation_id'],
                    'uuid': server['id'],
                    'state': 'error',
                    'power_state': 'shutdown',
                },
                actual=self.notifier.versioned_notifications[1])

            # Reboot server
            self.notifier.reset()

            post = {'reboot': {'type': 'HARD'}}
            self.api.post_server_action(server['id'], post)
            self._wait_for_notification('instance.reboot.start')
            self._wait_for_notification('instance.reboot.end')

        # Attach a share again in order to simulate a detach error
        self.manila_fixture.mock_allow.side_effect = original_side_effect
        self.api.post_server_action(server['id'], {'os-stop': {}})
        self._wait_for_state_change(server, expected_status='SHUTOFF')
        self.notifier.reset()

        self._attach_share(server, "e8debdc0-447a-4376-a10a-4cd9122d7986")
        share_info = self._get_share(
            server, "e8debdc0-447a-4376-a10a-4cd9122d7986", admin=True
        )
        expected_shares[0]["nova_object.data"][
            "share_mapping_uuid"
        ] = share_info["uuid"]

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                            self.notifier.versioned_notifications)
        expected_shares[0]['nova_object.data']['status'] = 'attaching'
        self._verify_notification(
            'instance-share_attach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[0])
        expected_shares[0]['nova_object.data']['status'] = 'inactive'
        self._verify_notification(
            'instance-share_attach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares
            },
            actual=self.notifier.versioned_notifications[1])

        # Simulate an error detaching the share.
        self.manila_fixture.mock_deny.side_effect = (
            exception.ShareAccessRemovalError(
                share_id="8db0037b-e98f-4bde-ae71-f96a077c19a4",
                reason="Connection timed out"
            )
        )
        self.notifier.reset()
        self._detach_share_with_error(
            server, "e8debdc0-447a-4376-a10a-4cd9122d7986")

        # 0: instance-share_detach-start
        # 1: instance-share_detach-error
        # 2: compute.exception
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                            self.notifier.versioned_notifications)
        expected_shares[0]["nova_object.data"]["status"] = "detaching"
        self._verify_notification(
            'instance-share_detach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares,
            },
            actual=self.notifier.versioned_notifications[0])
        expected_shares[0]["nova_object.data"]["status"] = "error"
        self._verify_notification(
            'instance-share_detach-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'state': 'stopped',
                'power_state': 'shutdown',
                'shares': expected_shares,
                'fault.traceback': self.ANY
            },
            actual=self.notifier.versioned_notifications[1])

        self._wait_for_notification('compute.exception')

        # Reboot server
        self.notifier.reset()

        post = {'reboot': {'type': 'HARD'}}
        self.api.post_server_action(server['id'], post)
        self._wait_for_notification('instance.reboot.start')
        self._wait_for_notification('instance.reboot.end')

    def _test_rescue_unrescue_server(self, server):
        # Both "rescue" and "unrescue" notification asserts are made here
        # rescue notification asserts
        post = {
            "rescue": {
                "rescue_image_ref": 'a2459075-d96c-40d5-893e-577ff92e721c'
            }
        }
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'RESCUE')

        # 0. instance.rescue.start
        # 1. instance.exists
        # 2. instance.rescue.end
        self.assertEqual(3, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-rescue-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-rescue-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[2])
        self.notifier.reset()

        # unrescue notification asserts
        post = {
            'unrescue': None
        }
        self.api.post_server_action(server['id'], post)
        self._wait_for_state_change(server, 'ACTIVE')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-unrescue-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-unrescue-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def _test_soft_delete_server(self, server):
        self.flags(reclaim_instance_interval=30)
        self.api.delete_server(server['id'])
        self._wait_for_state_change(server, 'SOFT_DELETED')

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-soft_delete-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-soft_delete-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])
        self.flags(reclaim_instance_interval=0)
        # Leave instance in normal, active state
        self.api.post_server_action(server['id'], {'restore': {}})

    def _test_attach_volume_error(self, server):
        def attach_volume(*args, **kwargs):
            raise exception.CinderConnectionFailed(
                reason="Connection timed out")

        # Override the fixture's default implementation of this with our
        # error-generating version.
        cinder.API.attachment_update.side_effect = attach_volume

        post = {"volumeAttachment": {"volumeId": self.cinder.SWAP_NEW_VOL}}
        self.api.post_server_volume(server['id'], post)

        self._wait_for_notification('instance.volume_attach.error')

        block_devices = [
            # Add by default at boot
            {'nova_object.data': {'boot_index': None,
                                  'delete_on_termination': False,
                                  'tag': None,
                                  'device_name': '/dev/sdb',
                                  'volume_id': self.cinder.SWAP_OLD_VOL},
             'nova_object.name': 'BlockDevicePayload',
             'nova_object.namespace': 'nova',
             'nova_object.version': '1.0'},
            # Attaching it right now
            {'nova_object.data': {'boot_index': None,
                                  'delete_on_termination': False,
                                  'tag': None,
                                  'device_name': '/dev/sdc',
                                  'volume_id': self.cinder.SWAP_NEW_VOL},
             'nova_object.name': 'BlockDevicePayload',
             'nova_object.namespace': 'nova',
             'nova_object.version': '1.0'}]

        # 0. volume_attach-start
        # 1. volume_attach-error
        # 2. compute.exception
        # We only rely on the first 2 notifications, in this case we don't
        # care about the exception notification.
        self.assertLessEqual(2, len(self.notifier.versioned_notifications))
        self._verify_notification(
            'instance-volume_attach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'block_devices': block_devices,
                'volume_id': self.cinder.SWAP_NEW_VOL,
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])

        tb = self.notifier.versioned_notifications[1]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn("CinderConnectionFailed:", tb)

        self._verify_notification(
            'instance-volume_attach-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'block_devices': block_devices,
                'volume_id': self.cinder.SWAP_NEW_VOL,
                'uuid': server['id'],
                'fault.traceback': self.ANY},
            actual=self.notifier.versioned_notifications[1])

    def _test_interface_attach_and_detach(self, server):
        post = {
            'interfaceAttachment': {
                'net_id': fixtures.NeutronFixture.network_1['id']
            }
        }
        self.api.attach_interface(server['id'], post)
        self._wait_for_notification('instance.interface_attach.end')
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-interface_attach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-interface_attach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

        self.notifier.reset()
        self.assertEqual(0, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)

        self.api.detach_interface(
            server['id'],
            fixtures.NeutronFixture.port_2['id'])
        self._wait_for_notification('instance.interface_detach.end')
        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-interface_detach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-interface_detach-end',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    @mock.patch('nova.virt.fake.SmallFakeDriver.attach_interface')
    def _test_interface_attach_error(self, server, mock_driver):
        def _unsuccessful_attach_interface(*args, **kwargs):
            raise exception.InterfaceAttachFailed("dummy")
        mock_driver.side_effect = _unsuccessful_attach_interface
        post = {
            'interfaceAttachment': {
                'net_id': fixtures.NeutronFixture.network_1['id']
            }
        }
        self.assertRaises(
            client.OpenStackApiException,
            self.api.attach_interface, server['id'], post)
        self._wait_for_notification('instance.interface_attach.error')
        # 0. instance.interface_attach.start
        # 1. instance.interface_attach.error
        # 2. compute.exception
        self.assertLessEqual(2, len(self.notifier.versioned_notifications))
        self._verify_notification(
            'instance-interface_attach-start',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])

        tb = self.notifier.versioned_notifications[1]['payload'][
            'nova_object.data']['fault']['nova_object.data']['traceback']
        self.assertIn("raise exception.InterfaceAttachFailed", tb)

        self._verify_notification(
            'instance-interface_attach-error',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id'],
                'fault.traceback': self.ANY},
            actual=self.notifier.versioned_notifications[1])

    def _test_lock_unlock_instance(self, server):
        self.api.post_server_action(server['id'], {'lock': {}})
        self._wait_for_server_parameter(server, {'locked': True})
        self.api.post_server_action(server['id'], {'unlock': {}})
        self._wait_for_server_parameter(server, {'locked': False})
        # Two versioned notifications are generated
        # 0. instance-lock
        # 1. instance-unlock

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-lock',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-unlock',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])

    def _test_lock_unlock_instance_with_reason(self, server):
        self.api.post_server_action(
            server['id'], {'lock': {"locked_reason": "global warming"}})
        self._wait_for_server_parameter(server, {'locked': True})
        self.api.post_server_action(server['id'], {'unlock': {}})
        self._wait_for_server_parameter(server, {'locked': False})
        # Two versioned notifications are generated
        # 0. instance-lock
        # 1. instance-unlock

        self.assertEqual(2, len(self.notifier.versioned_notifications),
                         self.notifier.versioned_notifications)
        self._verify_notification(
            'instance-lock-with-reason',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[0])
        self._verify_notification(
            'instance-unlock',
            replacements={
                'reservation_id': server['reservation_id'],
                'uuid': server['id']},
            actual=self.notifier.versioned_notifications[1])
