Merge "Update driver to map the targeted address for SR-IOV PCI devices"

nova/conf/pci.py

@@ -36,7 +36,24 @@ to use move operations, for each ``nova-compute`` service.
 
 Possible Values:
 
-* A dictionary of JSON values which describe the aliases. For example::
+* A JSON dictionary which describe a PCI device. It should take
+  the following format::
+
+    alias = {
+      "name": "<name>",
+      ["product_id": "<id>"],
+      ["vendor_id": "<id>"],
+      "device_type": "<type>",
+      ["numa_policy": "<policy>"],
+      ["resource_class": "<resource_class>"],
+      ["traits": "<traits>"]
+      ["live_migratable": "<live_migratable>"],
+    }
+
+  Where ``[`` indicates zero or one occurrences, ``{`` indicates zero or
+  multiple occurrences, and ``|`` mutually exclusive options.
+
+  For example::
 
     alias = {
       "name": "QuickAssist",
@@ -46,8 +63,17 @@ Possible Values:
       "numa_policy": "required"
     }
 
-  This defines an alias for the Intel QuickAssist card. (multi valued). Valid
-  key values are :
+  This defines an alias for the Intel QuickAssist card. (multi valued).
+
+  Another example::
+
+    alias = {
+      "name": "A16_16A",
+      "device_type": "type-VF",
+      resource_class: "CUSTOM_A16_16A",
+    }
+
+  Valid key values are :
 
   ``name``
     Name of the PCI alias.
@@ -97,6 +123,22 @@ Possible Values:
     scheduling the request. This field can only be used only if
     ``[filter_scheduler]pci_in_placement`` is enabled.
 
+  ``live_migratable``
+    Specify if live-migratable devices are desired.
+    May have boolean-like string values case-insensitive values:
+    "yes" or "no".
+
+    - ``live_migratable='yes'`` means that the user wants a device(s)
+      allowing live migration to a similar device(s) on another host.
+
+    - ``live_migratable='no'`` This explicitly indicates that the user
+      requires a non-live migratable device, making migration impossible.
+
+    - If not specified, the default is ``live_migratable=None``, meaning that
+      either a live migratable or non-live migratable device will be picked
+      automatically. However, in such cases, migration will **not** be
+      possible.
+
 * Supports multiple aliases by repeating the option (not by specifying
   a list value)::
 
@@ -112,7 +154,8 @@ Possible Values:
       "product_id": "0444",
       "vendor_id": "8086",
       "device_type": "type-PCI",
-      "numa_policy": "required"
+      "numa_policy": "required",
+      "live_migratable": "yes",
     }
 """),
     cfg.MultiStrOpt('device_spec',
@@ -165,7 +208,9 @@ Possible values:
     Supported ``<tag>`` values are :
 
     - ``physical_network``
+
     - ``trusted``
+
     - ``remote_managed`` - a VF is managed remotely by an off-path networking
       backend. May have boolean-like string values case-insensitive values:
       "true" or "false". By default, "false" is assumed for all devices.
@@ -174,6 +219,7 @@ Possible values:
       VPD capability with a card serial number (either on a VF itself on
       its corresponding PF), otherwise they will be ignored and not
       available for allocation.
+
     - ``managed`` - Specify if the PCI device is managed by libvirt.
       May have boolean-like string values case-insensitive values:
       "yes" or "no". By default, "yes" is assumed for all devices.
@@ -189,6 +235,18 @@ Possible values:
 
        Warning: Incorrect configuration of this parameter may result in compute
        node crashes.
+
+    - ``live_migratable`` - Specify if the PCI device is live_migratable by
+      libvirt.
+      May have boolean-like string values case-insensitive values:
+      "yes" or "no". By default, "no" is assumed for all devices.
+
+      - ``live_migratable='yes'`` means that the device can be live migrated.
+        Of course, this requires hardware support, as well as proper system
+        and hypervisor configuration.
+
+      - ``live_migratable='no'`` means that the device cannot be live migrated.
+
     - ``resource_class`` - optional Placement resource class name to be used
       to track the matching PCI devices in Placement when
       [pci]report_in_placement is True.
@@ -202,6 +260,7 @@ Possible values:
       device's ``vendor_id`` and ``product_id`` in the form of
       ``CUSTOM_PCI_{vendor_id}_{product_id}``.
       The ``resource_class`` can be requested from a ``[pci]alias``
+
     - ``traits`` - optional comma separated list of Placement trait names to
       report on the resource provider that will represent the matching PCI
       device. Each trait can be a standard trait from ``os-traits`` lib or can

nova/tests/unit/virt/libvirt/test_migration.py

@@ -35,6 +35,14 @@ from nova.virt.libvirt import host
 from nova.virt.libvirt import migration
 
 
+def _normalize(xml_str):
+    return etree.tostring(
+        etree.fromstring(xml_str),
+        pretty_print=True,
+        encoding="unicode",
+    ).strip()
+
+
 class UtilityMigrationTestCase(test.NoDBTestCase):
 
     def test_graphics_listen_addrs(self):
@@ -278,6 +286,193 @@ class UtilityMigrationTestCase(test.NoDBTestCase):
         self.assertRaises(exception.NovaException,
                           migration._update_mdev_xml, doc, data.target_mdevs)
 
+    def test_update_pci_dev_xml(self):
+
+        xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x00' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        expected_xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x26' slot='0x01' function='0x5'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        data = objects.LibvirtLiveMigrateData(
+            pci_dev_map_src_dst={"0000:25:00.4": "0000:26:01.5"})
+        doc = etree.fromstring(xml_pattern)
+        res = migration._update_pci_dev_xml(doc, data.pci_dev_map_src_dst)
+        self.assertEqual(
+            _normalize(expected_xml_pattern),
+            etree.tostring(res, encoding="unicode", pretty_print=True).strip(),
+        )
+
+    def test_update_pci_dev_xml_with_2_hostdevs(self):
+
+        xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x00' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x01' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x06' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        expected_xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x26' slot='0x01' function='0x5'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x26' slot='0x01' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x06' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        data = objects.LibvirtLiveMigrateData(
+            pci_dev_map_src_dst={
+                "0000:25:00.4": "0000:26:01.5",
+                "0000:25:01.4": "0000:26:01.4",
+            }
+        )
+        doc = etree.fromstring(xml_pattern)
+        res = migration._update_pci_dev_xml(doc, data.pci_dev_map_src_dst)
+        self.assertEqual(
+            _normalize(expected_xml_pattern),
+            etree.tostring(res, encoding="unicode", pretty_print=True).strip(),
+        )
+
+    def test_update_pci_dev_xml_with_2_hostdevs_second_one_not_in_map(self):
+
+        xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x00' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x01' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x06' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        expected_xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x26' slot='0x01' function='0x5'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x01' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x06' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        data = objects.LibvirtLiveMigrateData(
+            pci_dev_map_src_dst={
+                "0000:25:00.4": "0000:26:01.5",
+            }
+        )
+        doc = etree.fromstring(xml_pattern)
+        res = migration._update_pci_dev_xml(doc, data.pci_dev_map_src_dst)
+        self.assertEqual(
+            _normalize(expected_xml_pattern),
+            etree.tostring(res, encoding="unicode", pretty_print=True).strip(),
+        )
+
+    def test_update_pci_dev_xml_fails_not_found_src_address(self):
+        xml_pattern = """<domain>
+  <devices>
+    <hostdev mode='subsystem' type='pci' managed='no'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x0000' bus='0x25' slot='0x00' function='0x4'/>
+      </source>
+      <alias name='hostdev0'/>
+      <address type='pci' domain='0x0000' bus='0x00'
+      slot='0x05' function='0x0'/>
+    </hostdev>
+  </devices>
+</domain>"""
+        data = objects.LibvirtLiveMigrateData(
+            pci_dev_map_src_dst={"0000:25:00.5": "0000:26:01.5"})
+        doc = etree.fromstring(xml_pattern)
+        exc = self.assertRaises(
+            exception.NovaException,
+            migration._update_pci_dev_xml,
+            doc,
+            data.pci_dev_map_src_dst,
+        )
+
+        norm = _normalize(xml_pattern)
+
+        self.assertIn(
+            'Unable to find the hostdev '
+            f'to replace for this source PCI address: 0000:25:00.5 '
+            f'in the xml: {norm}',
+            str(exc),
+        )
+
     def test_update_cpu_shared_set_xml(self):
         doc = etree.fromstring("""
             <domain>

nova/virt/libvirt/driver.py

@@ -97,6 +97,7 @@ from nova.objects import diagnostics as diagnostics_obj
 from nova.objects import fields
 from nova.objects import migrate_data as migrate_data_obj
 from nova.pci import utils as pci_utils
+from nova.pci import whitelist
 import nova.privsep.libvirt
 import nova.privsep.path
 import nova.privsep.utils
@@ -266,6 +267,10 @@ MIN_LIBVIRT_STATELESS_FIRMWARE = (8, 6, 0)
 MIN_IGB_LIBVIRT_VERSION = (9, 3, 0)
 MIN_IGB_QEMU_VERSION = (8, 0, 0)
 
+# Minimum versions supporting vfio-pci variant driver.
+MIN_VFIO_PCI_VARIANT_LIBVIRT_VERSION = (10, 0, 0)
+MIN_VFIO_PCI_VARIANT_QEMU_VERSION = (8, 2, 2)
+
 REGISTER_IMAGE_PROPERTY_DEFAULTS = [
     'hw_machine_type',
     'hw_cdrom_bus',
@@ -902,10 +907,35 @@ class LibvirtDriver(driver.ComputeDriver):
 
         self._check_multipath()
 
+        # Even if we already checked the whitelist at startup, this driver
+        # needs to check specific hypervisor versions
+        self._check_pci_whitelist()
+
         # Set REGISTER_IMAGE_PROPERTY_DEFAULTS in the instance system_metadata
         # to default values for properties that have not already been set.
         self._register_all_undefined_instance_details()
 
+    def _check_pci_whitelist(self):
+
+        need_specific_version = False
+
+        if CONF.pci.device_spec:
+            pci_whitelist = whitelist.Whitelist(CONF.pci.device_spec)
+            for spec in pci_whitelist.specs:
+                if spec.tags.get("live_migratable"):
+                    need_specific_version = True
+
+        if need_specific_version and not self._host.has_min_version(
+            lv_ver=MIN_VFIO_PCI_VARIANT_LIBVIRT_VERSION,
+            hv_ver=MIN_VFIO_PCI_VARIANT_QEMU_VERSION,
+            hv_type=host.HV_DRIVER_QEMU,
+        ):
+            msg = _(
+                "PCI device spec is configured for "
+                "live_migratable but it's not supported by libvirt."
+            )
+            raise exception.InvalidConfiguration(msg)
+
     def _update_host_specific_capabilities(self) -> None:
         """Update driver capabilities based on capabilities of the host."""
         # TODO(stephenfin): We should also be reporting e.g. SEV functionality

nova/virt/libvirt/migration.py

@@ -16,7 +16,6 @@
 """Utility methods to manage guests migration
 
 """
-
 from collections import deque
 
 from lxml import etree
@@ -88,6 +87,11 @@ def get_updated_guest_xml(instance, guest, migrate_data, get_volume_config,
         xml_doc = _update_numa_xml(xml_doc, migrate_data)
     if 'target_mdevs' in migrate_data:
         xml_doc = _update_mdev_xml(xml_doc, migrate_data.target_mdevs)
+    if "pci_dev_map_src_dst" in migrate_data:
+        xml_doc = _update_pci_dev_xml(
+            xml_doc, migrate_data.pci_dev_map_src_dst
+        )
+
     if new_resources:
         xml_doc = _update_device_resources_xml(xml_doc, new_resources)
     return etree.tostring(xml_doc, encoding='unicode')
@@ -149,6 +153,77 @@ def _update_mdev_xml(xml_doc, target_mdevs):
     return xml_doc
 
 
+def _update_pci_dev_xml(xml_doc, pci_dev_map_src_dst):
+    hostdevs = xml_doc.findall('./devices/hostdev')
+
+    for src_addr, dst_addr in pci_dev_map_src_dst.items():
+        src_fields = _get_pci_address_fields_with_prefix(src_addr)
+        dst_fields = _get_pci_address_fields_with_prefix(dst_addr)
+
+        if not _update_hostdev_address(hostdevs, src_fields, dst_fields):
+            _raise_hostdev_not_found_exception(xml_doc, src_addr)
+
+    LOG.debug(
+        '_update_pci_xml output xml=%s',
+        etree.tostring(xml_doc, encoding='unicode', pretty_print=True)
+    )
+    return xml_doc
+
+
+def _get_pci_address_fields_with_prefix(addr):
+    (domain, bus, slot, func) = nova.pci.utils.get_pci_address_fields(addr)
+    return (f"0x{domain}", f"0x{bus}", f"0x{slot}", f"0x{func}")
+
+
+def _update_hostdev_address(hostdevs, src_fields, dst_fields):
+    src_domain, src_bus, src_slot, src_function = src_fields
+    dst_domain, dst_bus, dst_slot, dst_function = dst_fields
+
+    for hostdev in hostdevs:
+        if hostdev.get('type') != 'pci':
+            continue
+
+        address_tag = hostdev.find('./source/address')
+        if address_tag is None:
+            continue
+
+        if _address_matches(
+            address_tag, src_domain, src_bus, src_slot, src_function
+        ):
+            _set_address_fields(
+                address_tag, dst_domain, dst_bus, dst_slot, dst_function
+            )
+            return True
+
+    return False
+
+
+def _address_matches(address_tag, domain, bus, slot, function):
+    return (
+        address_tag.get('domain') == domain and
+        address_tag.get('bus') == bus and
+        address_tag.get('slot') == slot and
+        address_tag.get('function') == function
+    )
+
+
+def _set_address_fields(address_tag, domain, bus, slot, function):
+    address_tag.set('domain', domain)
+    address_tag.set('bus', bus)
+    address_tag.set('slot', slot)
+    address_tag.set('function', function)
+
+
+def _raise_hostdev_not_found_exception(xml_doc, src_addr):
+    xml = etree.tostring(
+        xml_doc, encoding="unicode", pretty_print=True
+    ).strip()
+    raise exception.NovaException(
+        'Unable to find the hostdev to replace for this source PCI '
+        f'address: {src_addr} in the xml: {xml}'
+    )
+
+
 def _update_cpu_shared_set_xml(xml_doc, migrate_data):
     LOG.debug('_update_cpu_shared_set_xml input xml=%s',
               etree.tostring(xml_doc, encoding='unicode', pretty_print=True))

releasenotes/notes/migrate-vfio-devices-using-kernel-variant-drivers-d4180849f973012e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    This release adds support for migrating SR-IOV devices
+    using the new kernel VFIO SR-IOV variant driver interface.
+    See the `OpenStack configuration documentation`__ for more details.
+
+    .. __: https://docs.openstack.org/nova/latest/configuration/config.html#pci