diff --git a/nova/tests/functional/libvirt/test_server_shares.py b/nova/tests/functional/libvirt/test_server_shares.py index 79feb50c38..b444434a63 100644 --- a/nova/tests/functional/libvirt/test_server_shares.py +++ b/nova/tests/functional/libvirt/test_server_shares.py @@ -12,20 +12,24 @@ import fixtures from lxml import etree +import os from requests import request from nova import context as nova_context +from nova import exception from nova.objects import instance +from nova.objects import share_mapping from nova.tests import fixtures as nova_fixtures +from nova.tests.functional.api import client from nova.tests.functional.libvirt import base +from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging from oslo_serialization import jsonutils from unittest import mock - CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -136,6 +140,76 @@ class ServerSharesTest(ServerSharesTestBase): self._get_metadata_url(server), share_id, share_id) return (server, share_id) + def test_server_share_metadata_with_tag(self): + """Verify that share metadata are available with the provided tag""" + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'disconnect_volume' + ), mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'connect_volume' + ): + traits = self._get_provider_traits( + self.compute_rp_uuids[self.compute]) + for trait in ( + 'COMPUTE_STORAGE_VIRTIO_FS', 'COMPUTE_MEM_BACKING_FILE'): + self.assertIn(trait, traits) + server = self._create_server(networks='auto') + self._stop_server(server) + + share_id = '4b021746-d0eb-4031-92aa-23c3bec182cd' + tag = 'mytag' + self._attach_share(server, share_id, tag=tag) + self._start_server(server) + + # tag is the filesystem target directory. + # if post /server/{server_id}/share was called without a specific + # tag then the tag is the share id. + self._assert_filesystem_tag(self._get_xml(server), tag) + + self._assert_share_in_metadata( + self._get_metadata_url(server), share_id, tag) + return (server, share_id) + + def test_server_share_fails_with_tag_already_used(self): + """Verify that share create fails if we use an already assigned tag""" + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'disconnect_volume' + ), mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'connect_volume' + ): + traits = self._get_provider_traits( + self.compute_rp_uuids[self.compute]) + for trait in ( + 'COMPUTE_STORAGE_VIRTIO_FS', 'COMPUTE_MEM_BACKING_FILE'): + self.assertIn(trait, traits) + server = self._create_server(networks='auto') + self._stop_server(server) + + share_id = '4b021746-d0eb-4031-92aa-23c3bec182cd' + tag = 'mytag' + self._attach_share(server, share_id, tag=tag) + + share_id = '1457bd85-7e7f-4835-92de-47834e1516b5' + tag = 'mytag' + + exc = self.assertRaises( + client.OpenStackApiException, + self._attach_share, + server, + share_id, + tag=tag + ) + + self.assertEqual(409, exc.response.status_code) + self.assertIn( + "Share '1457bd85-7e7f-4835-92de-47834e1516b5' or " + "tag 'mytag' already associated to this server.", + str(exc.response.text), + ) + def test_server_cephfs_share_metadata(self): """Verify that cephfs share metadata are available""" with mock.patch( @@ -173,3 +247,268 @@ class ServerSharesTest(ServerSharesTestBase): self._assert_share_in_metadata( self._get_metadata_url(server), share_id, share_id) return (server, share_id) + + def test_server_share_after_hard_reboot(self): + """Verify that share is still available after a reboot""" + server, share_id = self.test_server_share_metadata() + self._reboot_server(server, hard=True) + + self._assert_filesystem_tag(self._get_xml(server), share_id) + + self._assert_share_in_metadata( + self._get_metadata_url(server), share_id, share_id) + + def test_server_share_mount_failure(self): + os.environ['OS_DEBUG'] = "true" + traits = self._get_provider_traits(self.compute_rp_uuids[self.compute]) + for trait in ('COMPUTE_STORAGE_VIRTIO_FS', 'COMPUTE_MEM_BACKING_FILE'): + self.assertIn(trait, traits) + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver' + '.connect_volume', + side_effect=processutils.ProcessExecutionError + ): + server = self._create_server(networks='auto') + self._stop_server(server) + + share_id = '4b021746-d0eb-4031-92aa-23c3bec182cd' + + self._attach_share(server, share_id) + response = self._get_share(server, share_id) + + self.assertEqual( + response["share_id"], "4b021746-d0eb-4031-92aa-23c3bec182cd" + ) + self.assertEqual(response["status"], "inactive") + + # Here we are using CastAsCallFixture so we got an exception from + # nova compute. This should not happen without the fixture and + # the api should answer with a 201 status code. + exc = self.assertRaises( + client.OpenStackApiException, + self._start_server, + server + ) + + self.assertIn("nova.exception.ShareMountError", str(exc)) + + log_out = self.stdlog.logger.output + + self.assertIn( + "Share id 4b021746-d0eb-4031-92aa-23c3bec182cd mount error " + "from server", + log_out) + + sm = share_mapping.ShareMapping.get_by_instance_uuid_and_share_id( + self.context, server['id'], share_id) + self.assertEqual(sm.status, 'error') + self.instance = instance.Instance.get_by_uuid( + self.context, server['id']) + self.assertEqual(self.instance.vm_state, 'error') + return (server, share_id) + + def test_server_start_fails_share_in_error(self): + """Ensure a server can not start if its attached share is in error + status and hard reboot attempts allow to restart the server as + soon as the share issue is fixed. + """ + server, share_id = self.test_server_share_mount_failure() + self._verify_start_fails_share_in_error(server, share_id) + + server, share_id = self.test_server_share_umount_failure() + self._verify_start_fails_share_in_error(server, share_id) + + def _verify_start_fails_share_in_error(self, server, share_id): + exc = self.assertRaises( + client.OpenStackApiException, + self._start_server, + server + ) + + # Try to start a vm in error state. + self.assertEqual(exc.response.status_code, 409) + self.assertIn("Cannot \'start\' instance", exc.response.text) + self.assertIn("while it is in vm_state error", exc.response.text) + + # Reboot to do another mount attempt and fix the error. + # But the error is not fixed. + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver' + '.connect_volume', + side_effect=processutils.ProcessExecutionError + ): + + # Here we are using CastAsCallFixture so we got an exception from + # nova api. This should not happen without the fixture. + exc = self.assertRaises( + client.OpenStackApiException, + self._reboot_server, + server, + hard=True + ) + + log_out = self.stdlog.logger.output + + self.assertIn( + "Share id 4b021746-d0eb-4031-92aa-23c3bec182cd mount error " + "from server", + log_out) + + sm = share_mapping.ShareMapping.get_by_instance_uuid_and_share_id( + self.context, server['id'], share_id) + self.assertEqual(sm.status, 'error') + self.instance = instance.Instance.get_by_uuid( + self.context, server['id']) + self.assertEqual(self.instance.vm_state, 'error') + + # Reboot to do another mount attempt and fix the error. + # But the error is fixed now. + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver' + '.connect_volume', + ): + self._reboot_server(server, hard=True) + + sm = share_mapping.ShareMapping.get_by_instance_uuid_and_share_id( + self.context, server['id'], share_id) + self.assertEqual(sm.status, 'active') + self.instance = instance.Instance.get_by_uuid( + self.context, server['id']) + self.assertEqual(self.instance.vm_state, 'active') + + def test_detach_server_share_in_error(self): + """Ensure share can still be detached even if + the share are in an error state. + """ + os.environ['OS_DEBUG'] = "true" + server, share_id = self.test_server_share_umount_failure() + + # Simulate an attempt to detach that fail due to umount error. + # In that case we should have an umount error. + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver' + '.disconnect_volume', + side_effect=processutils.ProcessExecutionError + ): + + # Ensure we failed to umount he share + log_out = self.stdlog.logger.output + + self.assertIn( + "Share id 4b021746-d0eb-4031-92aa-23c3bec182cd umount error", + log_out) + + # We detach the share. As a consequence, we are leaking the share + # mounted on the compute. + self._detach_share(server, share_id) + + # Share is removed so not anymore in the DB. + self.assertRaises( + exception.ShareNotFound, + share_mapping.ShareMapping.get_by_instance_uuid_and_share_id, + self.context, + server['id'], + share_id + ) + self.instance = instance.Instance.get_by_uuid( + self.context, server['id']) + self.assertEqual(self.instance.vm_state, 'error') + + # Reboot the server to restart it without the share. + self._reboot_server(server, hard=True) + + self.instance = instance.Instance.get_by_uuid( + self.context, server['id']) + self.assertEqual(self.instance.vm_state, 'active') + + @mock.patch('nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver' + '.disconnect_volume', + side_effect=processutils.ProcessExecutionError) + @mock.patch('nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver' + '.connect_volume') + def test_server_share_umount_failure(self, mock_mount, mock_umount): + os.environ['OS_DEBUG'] = "true" + traits = self._get_provider_traits(self.compute_rp_uuids[self.compute]) + for trait in ('COMPUTE_STORAGE_VIRTIO_FS', 'COMPUTE_MEM_BACKING_FILE'): + self.assertIn(trait, traits) + server = self._create_server(networks='auto') + self._stop_server(server) + + share_id = '4b021746-d0eb-4031-92aa-23c3bec182cd' + self._attach_share(server, share_id) + self._start_server(server) + + # Here we are using CastAsCallFixture so we got an exception from + # nova compute. This should not happen without the fixture and + # the api should answer with a 202 status code. + exc = self.assertRaises( + client.OpenStackApiException, + self._stop_server, + server, + ) + + self.assertIn("nova.exception.ShareUmountError", str(exc)) + + log_out = self.stdlog.logger.output + + self.assertIn( + "Share id 4b021746-d0eb-4031-92aa-23c3bec182cd umount error " + "from server", + log_out) + + sm = share_mapping.ShareMapping.get_by_instance_uuid_and_share_id( + self.context, server['id'], share_id) + self.assertEqual(sm.status, 'error') + self.instance = instance.Instance.get_by_uuid( + self.context, server['id']) + self.assertEqual(self.instance.vm_state, 'error') + return (server, share_id) + + def test_server_resume_with_shares(self): + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'disconnect_volume' + ), mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'connect_volume' + ): + server = self._create_server(networks='auto') + self._stop_server(server) + + share_id = '4b021746-d0eb-4031-92aa-23c3bec182cd' + self._attach_share(server, share_id) + self._start_server(server) + self.assertRaises( + client.OpenStackApiException, + self._suspend_server, + server, + ) + + def test_server_rescue_with_shares(self): + with mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'disconnect_volume' + ), mock.patch( + 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver.' + 'connect_volume' + ): + server = self._create_server(networks='auto') + self._stop_server(server) + + share_id = '4b021746-d0eb-4031-92aa-23c3bec182cd' + self._attach_share(server, share_id) + self._start_server(server) + self._rescue_server(server) + + self._assert_filesystem_tag(self._get_xml(server), share_id) + + self._assert_share_in_metadata( + self._get_metadata_url(server), share_id, share_id) + + self._unrescue_server(server) + + self._assert_filesystem_tag(self._get_xml(server), share_id) + + self._assert_share_in_metadata( + self._get_metadata_url(server), share_id, share_id) + return (server, share_id)