diff --git a/etc/nova/nova.conf.sample b/etc/nova/nova.conf.sample index 8ed3372116..a28e4fc1d8 100644 --- a/etc/nova/nova.conf.sample +++ b/etc/nova/nova.conf.sample @@ -1944,7 +1944,7 @@ #libvirt_vif_driver=nova.virt.libvirt.vif.LibvirtGenericVIFDriver # Libvirt handlers for remote volumes. (list value) -#libvirt_volume_drivers=iscsi=nova.virt.libvirt.volume.LibvirtISCSIVolumeDriver,local=nova.virt.libvirt.volume.LibvirtVolumeDriver,fake=nova.virt.libvirt.volume.LibvirtFakeVolumeDriver,rbd=nova.virt.libvirt.volume.LibvirtNetVolumeDriver,sheepdog=nova.virt.libvirt.volume.LibvirtNetVolumeDriver,nfs=nova.virt.libvirt.volume.LibvirtNFSVolumeDriver,aoe=nova.virt.libvirt.volume.LibvirtAOEVolumeDriver,glusterfs=nova.virt.libvirt.volume.LibvirtGlusterfsVolumeDriver,fibre_channel=nova.virt.libvirt.volume.LibvirtFibreChannelVolumeDriver,scality=nova.virt.libvirt.volume.LibvirtScalityVolumeDriver +#libvirt_volume_drivers=iscsi=nova.virt.libvirt.volume.LibvirtISCSIVolumeDriver,iser=nova.virt.libvirt.volume.LibvirtISERVolumeDriver,local=nova.virt.libvirt.volume.LibvirtVolumeDriver,fake=nova.virt.libvirt.volume.LibvirtFakeVolumeDriver,rbd=nova.virt.libvirt.volume.LibvirtNetVolumeDriver,sheepdog=nova.virt.libvirt.volume.LibvirtNetVolumeDriver,nfs=nova.virt.libvirt.volume.LibvirtNFSVolumeDriver,aoe=nova.virt.libvirt.volume.LibvirtAOEVolumeDriver,glusterfs=nova.virt.libvirt.volume.LibvirtGlusterfsVolumeDriver,fibre_channel=nova.virt.libvirt.volume.LibvirtFibreChannelVolumeDriver,scality=nova.virt.libvirt.volume.LibvirtScalityVolumeDriver # Override the default disk prefix for the devices attached to # a server, which is dependent on libvirt_type. (valid options @@ -2080,6 +2080,10 @@ # (integer value) #num_iscsi_scan_tries=3 +# number of times to rescan iSER target to find volume +# (integer value) +#num_iser_scan_tries=3 + # the RADOS client name for accessing rbd volumes (string # value) #rbd_user= @@ -2107,6 +2111,9 @@ # use multipath connection of the iSCSI volume (boolean value) #libvirt_iscsi_use_multipath=false +# use multipath connection of the iSER volume (boolean value) +#libvirt_iser_use_multipath=false + # Path or URL to Scality SOFS configuration file (string # value) #scality_sofs_config= diff --git a/nova/exception.py b/nova/exception.py index c7efe8e986..1bd000c705 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -500,6 +500,10 @@ class ISCSITargetNotFoundForVolume(NotFound): msg_fmt = _("No target id found for volume %(volume_id)s.") +class ISERTargetNotFoundForVolume(NotFound): + message = _("No target id found for volume %(volume_id)s.") + + class DiskNotFound(NotFound): msg_fmt = _("No disk at %(location)s") diff --git a/nova/tests/virt/libvirt/test_libvirt_volume.py b/nova/tests/virt/libvirt/test_libvirt_volume.py index 54216d918c..6dc6317d33 100644 --- a/nova/tests/virt/libvirt/test_libvirt_volume.py +++ b/nova/tests/virt/libvirt/test_libvirt_volume.py @@ -190,6 +190,17 @@ class LibvirtVolumeTestCase(test.TestCase): '-n', 'node.startup', '-v', 'automatic')] self.assertEqual(self.executes, expected_commands) + def iser_connection(self, volume, location, iqn): + return { + 'driver_volume_type': 'iser', + 'data': { + 'volume_id': volume['id'], + 'target_portal': location, + 'target_iqn': iqn, + 'target_lun': 1, + } + } + def sheepdog_connection(self, volume): return { 'driver_volume_type': 'sheepdog', @@ -374,6 +385,61 @@ class LibvirtVolumeTestCase(test.TestCase): self.assertEqual(tree.find('./source').get('dev'), mpdev_filepath) libvirt_driver.disconnect_volume(connection_info, 'vde') + def test_libvirt_kvm_iser_volume_with_multipath(self): + self.flags(libvirt_iser_use_multipath=True) + self.stubs.Set(os.path, 'exists', lambda x: True) + devs = ['/dev/mapper/sda', '/dev/mapper/sdb'] + self.stubs.Set(self.fake_conn, 'get_all_block_devices', lambda: devs) + libvirt_driver = volume.LibvirtISERVolumeDriver(self.fake_conn) + name = 'volume-00000001' + location = '10.0.2.15:3260' + iqn = 'iqn.2010-10.org.iser.openstack:%s' % name + vol = {'id': 1, 'name': name} + connection_info = self.iser_connection(vol, location, iqn) + mpdev_filepath = '/dev/mapper/foo' + connection_info['data']['device_path'] = mpdev_filepath + disk_info = { + "bus": "virtio", + "dev": "vde", + "type": "disk", + } + target_portals = ['fake_portal1', 'fake_portal2'] + libvirt_driver._get_multipath_device_name = lambda x: mpdev_filepath + conf = libvirt_driver.connect_volume(connection_info, disk_info) + tree = conf.format_dom() + self.assertEqual(tree.find('./source').get('dev'), mpdev_filepath) + libvirt_driver.disconnect_volume(connection_info, 'vde') + + def test_libvirt_kvm_iser_volume_with_multipath_getmpdev(self): + self.flags(libvirt_iser_use_multipath=True) + self.stubs.Set(os.path, 'exists', lambda x: True) + libvirt_driver = volume.LibvirtISERVolumeDriver(self.fake_conn) + name0 = 'volume-00000000' + location0 = '10.0.2.15:3260' + iqn0 = 'iqn.2010-10.org.iser.openstack:%s' % name0 + vol0 = {'id': 0, 'name': name0} + dev0 = '/dev/disk/by-path/ip-%s-iscsi-%s-lun-0' % (location0, iqn0) + name = 'volume-00000001' + location = '10.0.2.15:3260' + iqn = 'iqn.2010-10.org.iser.openstack:%s' % name + vol = {'id': 1, 'name': name} + dev = '/dev/disk/by-path/ip-%s-iscsi-%s-lun-1' % (location, iqn) + devs = [dev0, dev] + self.stubs.Set(self.fake_conn, 'get_all_block_devices', lambda: devs) + connection_info = self.iser_connection(vol, location, iqn) + mpdev_filepath = '/dev/mapper/foo' + disk_info = { + "bus": "virtio", + "dev": "vde", + "type": "disk", + } + target_portals = ['fake_portal1', 'fake_portal2'] + libvirt_driver._get_multipath_device_name = lambda x: mpdev_filepath + conf = libvirt_driver.connect_volume(connection_info, disk_info) + tree = conf.format_dom() + self.assertEqual(tree.find('./source').get('dev'), mpdev_filepath) + libvirt_driver.disconnect_volume(connection_info, 'vde') + def test_libvirt_nfs_driver(self): # NOTE(vish) exists is to make driver assume connecting worked mnt_base = '/mnt' diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 93c1a1417b..3c9ab991ba 100755 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -158,6 +158,7 @@ libvirt_opts = [ cfg.ListOpt('libvirt_volume_drivers', default=[ 'iscsi=nova.virt.libvirt.volume.LibvirtISCSIVolumeDriver', + 'iser=nova.virt.libvirt.volume.LibvirtISERVolumeDriver', 'local=nova.virt.libvirt.volume.LibvirtVolumeDriver', 'fake=nova.virt.libvirt.volume.LibvirtFakeVolumeDriver', 'rbd=nova.virt.libvirt.volume.LibvirtNetVolumeDriver', diff --git a/nova/virt/libvirt/volume.py b/nova/virt/libvirt/volume.py index cdb322dcd1..25d3e5aa29 100644 --- a/nova/virt/libvirt/volume.py +++ b/nova/virt/libvirt/volume.py @@ -18,6 +18,7 @@ """Volume drivers for libvirt.""" +import glob import hashlib import os import time @@ -43,6 +44,9 @@ volume_opts = [ cfg.IntOpt('num_iscsi_scan_tries', default=3, help='number of times to rescan iSCSI target to find volume'), + cfg.IntOpt('num_iser_scan_tries', + default=3, + help='number of times to rescan iSER target to find volume'), cfg.StrOpt('rbd_user', help='the RADOS client name for accessing rbd volumes'), cfg.StrOpt('rbd_secret_uuid', @@ -64,6 +68,9 @@ volume_opts = [ cfg.BoolOpt('libvirt_iscsi_use_multipath', default=False, help='use multipath connection of the iSCSI volume'), + cfg.BoolOpt('libvirt_iser_use_multipath', + default=False, + help='use multipath connection of the iSER volume'), cfg.StrOpt('scality_sofs_config', help='Path or URL to Scality SOFS configuration file'), cfg.StrOpt('scality_sofs_mount_point', @@ -478,6 +485,191 @@ class LibvirtISCSIVolumeDriver(LibvirtBaseVolumeDriver): self._run_multipath('-r', check_exit_code=[0, 1, 21]) +class LibvirtISERVolumeDriver(LibvirtISCSIVolumeDriver): + """Driver to attach Network volumes to libvirt.""" + + @utils.synchronized('connect_volume') + def connect_volume(self, connection_info, disk_info): + """Attach the volume to instance_name.""" + conf = LibvirtBaseVolumeDriver.connect_volume(self, connection_info, + disk_info) + + iser_properties = connection_info['data'] + + libvirt_iser_use_multipath = CONF.libvirt_iser_use_multipath + + if libvirt_iser_use_multipath: + # multipath installed, discovering other targets if available + # multipath should be configured on the nova-compute node, + # in order to fit storage vendor + out = self._run_iscsiadm_bare(['-m', + 'discovery', + '-t', + 'sendtargets', + '-p', + iser_properties['target_portal']], + check_exit_code=[0, 255])[0] or "" + + for ip in self._get_target_portals_from_iscsiadm_output(out): + props = iser_properties.copy() + props['target_portal'] = ip + self._connect_to_iser_portal(props) + + self._rescan_iscsi() + else: + self._connect_to_iser_portal(iser_properties) + + time.sleep(1) + host_device = None + device = ("ip-%s-iscsi-%s-lun-%s" % + (iser_properties['target_portal'], + iser_properties['target_iqn'], + iser_properties.get('target_lun', 0))) + look_for_device = glob.glob('/dev/disk/by-path/*%s' % device) + if look_for_device: + host_device = look_for_device[0] + + # The /dev/disk/by-path/... node is not always present immediately + tries = 0 + disk_dev = disk_info['dev'] + while not os.path.exists(host_device): + if tries >= CONF.num_iser_scan_tries: + raise exception.NovaException(_("iSER device not found at %s") + % host_device) + + LOG.warn(_("ISER volume not yet found at: %(disk_dev)s. " + "Will rescan & retry. Try number: %(tries)s") % + {'disk_dev': disk_dev, + 'tries': tries}) + + # The rescan isn't documented as being necessary(?), but it helps + self._run_iscsiadm(iser_properties, ("--rescan",)) + + tries = tries + 1 + if not os.path.exists(host_device): + time.sleep(tries ** 2) + + if tries != 0: + LOG.debug(_("Found iSER node %(disk_dev)s " + "(after %(tries)s rescans)") % + {'disk_dev': disk_dev, + 'tries': tries}) + + if libvirt_iser_use_multipath: + # we use the multipath device instead of the single path device + self._rescan_multipath() + multipath_device = self._get_multipath_device_name(host_device) + if multipath_device is not None: + host_device = multipath_device + + conf.source_type = "block" + conf.source_path = host_device + return conf + + @utils.synchronized('connect_volume') + def disconnect_volume(self, connection_info, disk_dev): + """Detach the volume from instance_name.""" + iser_properties = connection_info['data'] + multipath_device = None + if CONF.libvirt_iser_use_multipath: + host_device = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-%s" % + (iser_properties['target_portal'], + iser_properties['target_iqn'], + iser_properties.get('target_lun', 0))) + multipath_device = self._get_multipath_device_name(host_device) + + LibvirtBaseVolumeDriver.disconnect_volume(self, connection_info, + disk_dev) + + if CONF.libvirt_iser_use_multipath and multipath_device: + return self._disconnect_volume_multipath_iscsi(iser_properties) + + # NOTE(vish): Only disconnect from the target if no luns from the + # target are in use. + device_prefix = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-" % + (iser_properties['target_portal'], + iser_properties['target_iqn'])) + devices = self.connection.get_all_block_devices() + devices = [dev for dev in devices if dev.startswith(device_prefix)] + if not devices: + self._disconnect_from_iscsi_portal(iser_properties) + + def _connect_to_iser_portal(self, iser_properties): + # NOTE(vish): If we are on the same host as nova volume, the + # discovery makes the target so we don't need to + # run --op new. Therefore, we check to see if the + # target exists, and if we get 255 (Not Found), then + # we run --op new. This will also happen if another + # volume is using the same target. + try: + self._run_iscsiadm(iser_properties, ()) + except processutils.ProcessExecutionError as exc: + # iscsiadm returns 21 for "No records found" after version 2.0-871 + if exc.exit_code in [21, 255]: + self._run_iscsiadm(iser_properties, + ('--interface', 'iser', '--op', 'new')) + else: + raise + + if iser_properties.get('auth_method'): + self._iscsiadm_update(iser_properties, + "node.session.auth.authmethod", + iser_properties['auth_method']) + self._iscsiadm_update(iser_properties, + "node.session.auth.username", + iser_properties['auth_username']) + self._iscsiadm_update(iser_properties, + "node.session.auth.password", + iser_properties['auth_password']) + + self._iscsiadm_update(iser_properties, + "iface.transport_name", + "iser") + + # duplicate logins crash iscsiadm after load, + # so we scan active sessions to see if the node is logged in. + out = self._run_iscsiadm_bare(["-m", "session"], + run_as_root=True, + check_exit_code=[0, 1, 21])[0] or "" + + portals = [{'portal': p.split(" ")[2], 'iqn': p.split(" ")[3]} + for p in out.splitlines() if p.startswith("iser:")] + + stripped_portal = iser_properties['target_portal'].split(",")[0] + filtered_portal = [s for s in portals + if stripped_portal == + s['portal'].split(",")[0] + and + s['iqn'] == + iser_properties['target_iqn']] + if len(portals) == 0 or len(filtered_portal) == 0: + try: + self._run_iscsiadm(iser_properties, + ("--login",), + check_exit_code=[0, 255]) + except processutils.ProcessExecutionError as err: + # as this might be one of many paths, + # only set successfull logins to startup automatically + if err.exit_code == 15: + self._iscsiadm_update(iser_properties, + "node.startup", + "automatic") + return + + self._iscsiadm_update(iser_properties, + "node.startup", + "automatic") + + def _get_multipath_iqn(self, multipath_device): + entries = self._get_iscsi_devices() + for entry in entries: + entry_real_path = os.path.realpath("/dev/disk/by-path/%s" % entry) + entry_multipath = self._get_multipath_device_name(entry_real_path) + if entry_multipath == multipath_device: + return entry.split("iser-")[1].split("-lun")[0] + return None + + class LibvirtNFSVolumeDriver(LibvirtBaseVolumeDriver): """Class implements libvirt part of volume driver for NFS."""