From ab49f97b2c08294234c7bfd3dedb75780ca519e6 Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Thu, 30 Sep 2021 22:36:47 +0300 Subject: [PATCH] [yoga] Add PCI VPD Capability Handling This change comes as a part of the "Off-path Networking Backends Support" spec implementation. https://review.opendev.org/c/openstack/nova-specs/+/787458 * Add VPD capability parsing support * The XML data from libvirt is parsed and formatted into PCI device JSON dict that is sent to Nova API and is stored in the extra_info column of a PciDevice. The code gracefully handles the lack of the capability since it is optional or Libvirt may not support it in a particular release. https://libvirt.org/news.html#v7-9-0-2021-11-01 (VPD capability was added in 7.9.0). * Pass the serial number to Neutron in port updates If a card serial number is present based on the information from PCI VPD, pass it to Neutron along with other PCI-related information. Change-Id: I6445433142286728a8c7efadcf80d07082d60bc3 Implements: blueprint integration-with-off-path-network-backends --- nova/network/neutron.py | 24 ++- nova/objects/pci_device.py | 6 + nova/tests/fixtures/libvirt_data.py | 162 ++++++++++++++++++ nova/tests/unit/network/test_neutron.py | 77 +++++++-- nova/tests/unit/objects/test_pci_device.py | 10 ++ nova/tests/unit/virt/libvirt/test_config.py | 80 +++++++++ nova/tests/unit/virt/libvirt/test_host.py | 95 +++++++++- nova/virt/libvirt/config.py | 101 +++++++++++ nova/virt/libvirt/driver.py | 10 +- nova/virt/libvirt/host.py | 88 +++++++++- .../pci-vpd-capability-0d8039629db4afb8.yaml | 20 +++ 11 files changed, 642 insertions(+), 31 deletions(-) create mode 100644 releasenotes/notes/pci-vpd-capability-0d8039629db4afb8.yaml diff --git a/nova/network/neutron.py b/nova/network/neutron.py index e02da93b57..294ccaebbe 100644 --- a/nova/network/neutron.py +++ b/nova/network/neutron.py @@ -667,7 +667,8 @@ class API: # for the physical device but don't want to overwrite the other # information in the binding profile. for profile_key in ('pci_vendor_info', 'pci_slot', - constants.ALLOCATION, 'arq_uuid'): + constants.ALLOCATION, 'arq_uuid', + 'physical_network', 'card_serial_number'): if profile_key in port_profile: del port_profile[profile_key] port_req_body['port'][constants.BINDING_PROFILE] = port_profile @@ -1506,11 +1507,22 @@ class API: def _get_pci_device_profile(self, pci_dev): dev_spec = self.pci_whitelist.get_devspec(pci_dev) if dev_spec: - return {'pci_vendor_info': "%s:%s" % - (pci_dev.vendor_id, pci_dev.product_id), - 'pci_slot': pci_dev.address, - 'physical_network': - dev_spec.get_tags().get('physical_network')} + dev_profile = { + 'pci_vendor_info': "%s:%s" + % (pci_dev.vendor_id, pci_dev.product_id), + 'pci_slot': pci_dev.address, + 'physical_network': dev_spec.get_tags().get( + 'physical_network' + ), + } + if pci_dev.dev_type == obj_fields.PciDeviceType.SRIOV_VF: + card_serial_number = pci_dev.card_serial_number + if card_serial_number: + dev_profile.update({ + 'card_serial_number': card_serial_number + }) + return dev_profile + raise exception.PciDeviceNotFound(node_id=pci_dev.compute_node_id, address=pci_dev.address) diff --git a/nova/objects/pci_device.py b/nova/objects/pci_device.py index 0be94897e3..275d5da356 100644 --- a/nova/objects/pci_device.py +++ b/nova/objects/pci_device.py @@ -511,6 +511,12 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject): def is_available(self): return self.status == fields.PciDeviceStatus.AVAILABLE + @property + def card_serial_number(self): + caps_json = self.extra_info.get('capabilities', "{}") + caps = jsonutils.loads(caps_json) + return caps.get('vpd', {}).get('card_serial_number') + @base.NovaObjectRegistry.register class PciDeviceList(base.ObjectListBase, base.NovaObject): diff --git a/nova/tests/fixtures/libvirt_data.py b/nova/tests/fixtures/libvirt_data.py index 463cb0ae3f..43fcc8c950 100644 --- a/nova/tests/fixtures/libvirt_data.py +++ b/nova/tests/fixtures/libvirt_data.py @@ -2002,6 +2002,168 @@ _fake_NodeDevXml = { """, + # A PF with the VPD capability. + "pci_0000_82_00_0": """ + + pci_0000_82_00_0 + /sys/devices/pci0000:80/0000:80:03.0/0000:82:00.0 + pci_0000_80_03_0 + + mlx5_core + + + 0x020000 + 0 + 130 + 0 + 0 + MT42822 BlueField-2 integrated ConnectX-6 Dx network controller + Mellanox Technologies + +
+
+
+
+
+
+
+
+ + + BlueField-2 DPU 25GbE Dual-Port SFP56, Crypto Enabled, 16GB on-board DDR, 1GbE OOB management, Tall Bracket + + B1 + foobar + MBF2H332A-AEEOT + MT2113X00000 + PCIeGen4 x8 + MBF2H332A-AEEOT + 3c53d07eec484d8aab34dabd24fe575aa + MLX:MN=MLNX:CSKU=V2:UUID=V3:PCI=V0:MODL=BF2H332A + + + fooasset + vendorfield0 + vendorfield2 + vendorfieldA + systemfieldB + systemfield0 + + + +
+ + + + + + + + """, # noqa:E501 + # A VF without the VPD capability with a PF that has a VPD capability. + "pci_0000_82_00_3": """ + + pci_0000_82_00_3 + /sys/devices/pci0000:80/0000:80:03.0/0000:82:00.3 + pci_0000_80_03_0 + + mlx5_core + + + 0x020000 + 0 + 130 + 0 + 3 + ConnectX Family mlx5Gen Virtual Function + Mellanox Technologies + +
+ + +
+ + + + + + + + """, + # A VF with the VPD capability but without a parent defined in test data + # so that the VPD cap is extracted from the VF directly. + "pci_0001_82_00_3": """ + + pci_0001_82_00_3 + /sys/devices/pci0001:80/0001:80:03.0/0001:82:00.3 + pci_0001_80_03_0 + + mlx5_core + + + 0x020000 + 1 + 130 + 0 + 3 + ConnectX Family mlx5Gen Virtual Function + Mellanox Technologies + +
+ + + BlueField-2 DPU 25GbE Dual-Port SFP56, Crypto Enabled, 16GB on-board DDR, 1GbE OOB management, Tall Bracket + + B1 + MBF2H332A-AEEOT + MT2113XBEEF0 + MBF2H332A-AEEOT + 9644e3586190eb118000b8cef671bf3e + MLX:MN=MLNX:CSKU=V2:UUID=V3:PCI=V0:MODL=BF2H332A + PCIeGen4 x8 + + + +
+ + + + + + + + """, # noqa:E501 + # A VF without the VPD capability and without a parent PF defined + # in the test data. + "pci_0002_82_00_3": """ + + pci_0002_82_00_3 + /sys/devices/pci0002:80/0002:80:03.0/0002:82:00.3 + pci_0002_80_03_0 + + mlx5_core + + + 0x020000 + 2 + 130 + 0 + 3 + ConnectX Family mlx5Gen Virtual Function + Mellanox Technologies + +
+ + +
+ + + + + + + + """, # noqa:E501 } _fake_NodeDevXml_parents = { diff --git a/nova/tests/unit/network/test_neutron.py b/nova/tests/unit/network/test_neutron.py index 5056b70c4e..a06549b609 100644 --- a/nova/tests/unit/network/test_neutron.py +++ b/nova/tests/unit/network/test_neutron.py @@ -39,6 +39,7 @@ from nova.network import constants from nova.network import model from nova.network import neutron as neutronapi from nova import objects +from nova.objects import fields as obj_fields from nova.objects import network_request as net_req_obj from nova.objects import virtual_interface as obj_vif from nova.pci import manager as pci_manager @@ -4494,17 +4495,21 @@ class TestAPI(TestAPIBase): instance = fake_instance.fake_instance_obj(self.context) instance.migration_context = objects.MigrationContext() instance.migration_context.old_pci_devices = objects.PciDeviceList( - objects=[objects.PciDevice(vendor_id='1377', - product_id='0047', - address='0000:0a:00.1', - compute_node_id=1, - request_id='1234567890')]) + objects=[objects.PciDevice( + vendor_id='1377', + product_id='0047', + address='0000:0a:00.1', + compute_node_id=1, + request_id='1234567890', + dev_type=obj_fields.PciDeviceType.SRIOV_VF)]) instance.migration_context.new_pci_devices = objects.PciDeviceList( - objects=[objects.PciDevice(vendor_id='1377', - product_id='0047', - address='0000:0b:00.1', - compute_node_id=2, - request_id='1234567890')]) + objects=[objects.PciDevice( + vendor_id='1377', + product_id='0047', + address='0000:0b:00.1', + compute_node_id=2, + request_id='1234567890', + dev_type=obj_fields.PciDeviceType.SRIOV_VF)]) instance.pci_devices = instance.migration_context.old_pci_devices # Validate that non-direct port aren't updated (fake-port-2). @@ -5928,14 +5933,14 @@ class TestAPI(TestAPIBase): 'id': uuids.port, 'binding:profile': {'pci_vendor_info': '1377:0047', 'pci_slot': '0000:0a:00.1', + 'card_serial_number': 'MT2113X00000', 'physical_network': 'physnet1', 'capabilities': ['switchdev']} } self.api._unbind_ports(self.context, ports, neutron, port_client) port_req_body = {'port': {'binding:host_id': None, 'binding:profile': - {'physical_network': 'physnet1', - 'capabilities': ['switchdev']}, + {'capabilities': ['switchdev']}, 'device_id': '', 'device_owner': ''} } @@ -7402,9 +7407,12 @@ class TestAPIPortbinding(TestAPIBase): pci_dev = {'vendor_id': '1377', 'product_id': '0047', 'address': '0000:0a:00.1', + 'card_serial_number': None, + 'dev_type': 'TEST_TYPE', } PciDevice = collections.namedtuple('PciDevice', - ['vendor_id', 'product_id', 'address']) + ['vendor_id', 'product_id', 'address', + 'card_serial_number', 'dev_type']) mydev = PciDevice(**pci_dev) profile = {'pci_vendor_info': '1377:0047', 'pci_slot': '0000:0a:00.1', @@ -7422,6 +7430,43 @@ class TestAPIPortbinding(TestAPIBase): port_req_body['port'][ constants.BINDING_PROFILE]) + @mock.patch.object(pci_whitelist.Whitelist, 'get_devspec') + @mock.patch.object(pci_manager, 'get_instance_pci_devs') + def test_populate_neutron_extension_values_binding_sriov_card_serial( + self, mock_get_instance_pci_devs, mock_get_pci_device_devspec): + host_id = 'my_host_id' + instance = {'host': host_id} + port_req_body = {'port': {}} + pci_req_id = 'my_req_id' + pci_dev = {'vendor_id': 'a2d6', + 'product_id': '15b3', + 'address': '0000:82:00.1', + 'card_serial_number': 'MT2113X00000', + 'dev_type': obj_fields.PciDeviceType.SRIOV_VF, + } + PciDevice = collections.namedtuple('PciDevice', + ['vendor_id', 'product_id', 'address', + 'card_serial_number', 'dev_type']) + mydev = PciDevice(**pci_dev) + profile = {'pci_vendor_info': 'a2d6:15b3', + 'pci_slot': '0000:82:00.1', + 'physical_network': 'physnet1', + # card_serial_number is a property of the object obtained + # from extra_info. + 'card_serial_number': 'MT2113X00000', + } + + mock_get_instance_pci_devs.return_value = [mydev] + devspec = mock.Mock() + devspec.get_tags.return_value = {'physical_network': 'physnet1'} + mock_get_pci_device_devspec.return_value = devspec + self.api._populate_neutron_binding_profile( + instance, pci_req_id, port_req_body, None) + + self.assertEqual(profile, + port_req_body['port'][ + constants.BINDING_PROFILE]) + def test_populate_neutron_extension_values_binding_arq(self): host_id = 'my_host_id' instance = {'host': host_id} @@ -7474,9 +7519,12 @@ class TestAPIPortbinding(TestAPIBase): pci_dev = {'vendor_id': '1377', 'product_id': '0047', 'address': '0000:0a:00.1', + 'card_serial_number': None, + 'dev_type': 'TEST_TYPE', } PciDevice = collections.namedtuple('PciDevice', - ['vendor_id', 'product_id', 'address']) + ['vendor_id', 'product_id', 'address', + 'card_serial_number', 'dev_type']) mydev = PciDevice(**pci_dev) profile = {'pci_vendor_info': '1377:0047', 'pci_slot': '0000:0a:00.1', @@ -7547,6 +7595,7 @@ class TestAPIPortbinding(TestAPIBase): pci_dev = {'vendor_id': '1377', 'product_id': '0047', 'address': '0000:0a:00.1', + 'dev_type': obj_fields.PciDeviceType.SRIOV_VF, } whitelist = pci_whitelist.Whitelist(CONF.pci.passthrough_whitelist) diff --git a/nova/tests/unit/objects/test_pci_device.py b/nova/tests/unit/objects/test_pci_device.py index 277a7fe7c4..4087b89800 100644 --- a/nova/tests/unit/objects/test_pci_device.py +++ b/nova/tests/unit/objects/test_pci_device.py @@ -161,6 +161,16 @@ class _TestPciDeviceObject(object): 'vendor_id', 'numa_node', 'status', 'uuid', 'extra_info', 'dev_type', 'parent_addr'])) + def test_pci_device_extra_info_card_serial_number(self): + self.dev_dict = copy.copy(dev_dict) + self.pci_device = pci_device.PciDevice.create(None, self.dev_dict) + self.assertIsNone(self.pci_device.card_serial_number) + + self.dev_dict = copy.copy(dev_dict) + self.dev_dict['capabilities'] = {'vpd': {'card_serial_number': '42'}} + self.pci_device = pci_device.PciDevice.create(None, self.dev_dict) + self.assertEqual(self.pci_device.card_serial_number, '42') + def test_update_device(self): self.pci_device = pci_device.PciDevice.create(None, dev_dict) self.pci_device.obj_reset_changes() diff --git a/nova/tests/unit/virt/libvirt/test_config.py b/nova/tests/unit/virt/libvirt/test_config.py index 2d690e5dfc..396edfd024 100644 --- a/nova/tests/unit/virt/libvirt/test_config.py +++ b/nova/tests/unit/virt/libvirt/test_config.py @@ -3273,6 +3273,86 @@ class LibvirtConfigNodeDevicePciCapTest(LibvirtConfigBaseTest): 'name': 'GRID M60-0B', 'type': 'nvidia-11'}], obj.mdev_capability[0].mdev_types) + def test_config_device_pci_vpd(self): + xmlin = """ + + 0x020000 + 0 + 130 + 0 + 1 + MT42822 BlueField-2 + Mellanox Technologies + + + BlueField-2 DPU 25GbE + + B1 + foobar + MBF2H332A-AEEOT + MT2113X00000 + PCIeGen4 x8 + MBF2H332A-AEEOT + 3c53d07eec484d8aab34dabd24fe575aa + MLX:MN=MLNX:CSKU=V2:UUID=V3:PCI=V0:MODL=BF2H332A + + + fooasset + vendorfield0 + vendorfield2 + vendorfieldA + systemfieldB + systemfield0 + + + +
+ + + + + + + """ # noqa: E501 + obj = config.LibvirtConfigNodeDevicePciCap() + obj.parse_str(xmlin) + + # Asserting common PCI attribute parsing. + self.assertEqual(0, obj.domain) + self.assertEqual(130, obj.bus) + self.assertEqual(0, obj.slot) + self.assertEqual(1, obj.function) + # Asserting vpd capability parsing. + self.assertEqual("MT42822 BlueField-2", obj.product) + self.assertEqual(0xA2D6, obj.product_id) + self.assertEqual("Mellanox Technologies", obj.vendor) + self.assertEqual(0x15B3, obj.vendor_id) + self.assertEqual(obj.numa_node, 1) + self.assertIsInstance(obj.vpd_capability, + config.LibvirtConfigNodeDeviceVpdCap) + self.assertEqual(obj.vpd_capability.card_name, 'BlueField-2 DPU 25GbE') + + self.assertEqual(obj.vpd_capability.change_level, 'B1') + self.assertEqual(obj.vpd_capability.manufacture_id, 'foobar') + self.assertEqual(obj.vpd_capability.part_number, 'MBF2H332A-AEEOT') + self.assertEqual(obj.vpd_capability.card_serial_number, 'MT2113X00000') + self.assertEqual(obj.vpd_capability.asset_tag, 'fooasset') + self.assertEqual(obj.vpd_capability.ro_vendor_fields, { + '0': 'PCIeGen4 x8', + '2': 'MBF2H332A-AEEOT', + '3': '3c53d07eec484d8aab34dabd24fe575aa', + 'A': 'MLX:MN=MLNX:CSKU=V2:UUID=V3:PCI=V0:MODL=BF2H332A', + }) + self.assertEqual(obj.vpd_capability.rw_vendor_fields, { + '0': 'vendorfield0', + '2': 'vendorfield2', + 'A': 'vendorfieldA', + }) + self.assertEqual(obj.vpd_capability.rw_system_fields, { + '0': 'systemfield0', + 'B': 'systemfieldB', + }) + class LibvirtConfigNodeDevicePciSubFunctionCap(LibvirtConfigBaseTest): diff --git a/nova/tests/unit/virt/libvirt/test_host.py b/nova/tests/unit/virt/libvirt/test_host.py index 192909d721..a33ffef702 100644 --- a/nova/tests/unit/virt/libvirt/test_host.py +++ b/nova/tests/unit/virt/libvirt/test_host.py @@ -1119,7 +1119,7 @@ Active: 8381604 kB pci_dev = fakelibvirt.NodeDevice( self.host._get_connection(), xml=fake_libvirt_data._fake_NodeDevXml[dev_name]) - actual_vf = self.host._get_pcidev_info(dev_name, pci_dev, [], []) + actual_vf = self.host._get_pcidev_info(dev_name, pci_dev, [], [], []) expect_vf = { "dev_id": dev_name, "address": "0000:04:11.7", "product_id": '1520', "numa_node": 0, @@ -1135,7 +1135,9 @@ Active: 8381604 kB def test_get_pcidev_info(self, mock_get_ifname): devs = { "pci_0000_04_00_3", "pci_0000_04_10_7", "pci_0000_04_11_7", - "pci_0000_04_00_1", "pci_0000_03_00_0", "pci_0000_03_00_1" + "pci_0000_04_00_1", "pci_0000_03_00_0", "pci_0000_03_00_1", + "pci_0000_82_00_0", "pci_0000_82_00_3", "pci_0001_82_00_3", + "pci_0002_82_00_3", } node_devs = {} for dev_name in devs: @@ -1150,10 +1152,12 @@ Active: 8381604 kB xml=fake_libvirt_data._fake_NodeDevXml[child])) net_devs = [ dev for dev in node_devs.values() if dev.name() not in devs] + pci_devs = [ + dev for dev in node_devs.values() if dev.name() in devs] name = "pci_0000_04_00_3" actual_vf = self.host._get_pcidev_info( - name, node_devs[name], net_devs, []) + name, node_devs[name], net_devs, [], []) expect_vf = { "dev_id": "pci_0000_04_00_3", "address": "0000:04:00.3", @@ -1167,7 +1171,7 @@ Active: 8381604 kB name = "pci_0000_04_10_7" actual_vf = self.host._get_pcidev_info( - name, node_devs[name], net_devs, []) + name, node_devs[name], net_devs, [], []) expect_vf = { "dev_id": "pci_0000_04_10_7", "address": "0000:04:10.7", @@ -1186,7 +1190,7 @@ Active: 8381604 kB name = "pci_0000_04_11_7" actual_vf = self.host._get_pcidev_info( - name, node_devs[name], net_devs, []) + name, node_devs[name], net_devs, [], []) expect_vf = { "dev_id": "pci_0000_04_11_7", "address": "0000:04:11.7", @@ -1205,7 +1209,7 @@ Active: 8381604 kB name = "pci_0000_04_00_1" actual_vf = self.host._get_pcidev_info( - name, node_devs[name], net_devs, []) + name, node_devs[name], net_devs, [], []) expect_vf = { "dev_id": "pci_0000_04_00_1", "address": "0000:04:00.1", @@ -1219,7 +1223,7 @@ Active: 8381604 kB name = "pci_0000_03_00_0" actual_vf = self.host._get_pcidev_info( - name, node_devs[name], net_devs, []) + name, node_devs[name], net_devs, [], []) expect_vf = { "dev_id": "pci_0000_03_00_0", "address": "0000:03:00.0", @@ -1233,7 +1237,7 @@ Active: 8381604 kB name = "pci_0000_03_00_1" actual_vf = self.host._get_pcidev_info( - name, node_devs[name], net_devs, []) + name, node_devs[name], net_devs, [], []) expect_vf = { "dev_id": "pci_0000_03_00_1", "address": "0000:03:00.1", @@ -1245,6 +1249,81 @@ Active: 8381604 kB } self.assertEqual(expect_vf, actual_vf) + # Parent PF with a VPD cap. + name = "pci_0000_82_00_0" + actual_pf = self.host._get_pcidev_info( + name, node_devs[name], net_devs, [], pci_devs) + expect_pf = { + "dev_id": "pci_0000_82_00_0", + "address": "0000:82:00.0", + "product_id": "a2d6", + "numa_node": 1, + "vendor_id": "15b3", + "label": "label_15b3_a2d6", + "dev_type": obj_fields.PciDeviceType.SRIOV_PF, + "capabilities": { + # Should be obtained from the parent PF in this case. + "vpd": {"card_serial_number": "MT2113X00000"}}, + } + self.assertEqual(expect_pf, actual_pf) + + # A VF without a VPD cap with a parent PF that has a VPD cap. + name = "pci_0000_82_00_3" + actual_vf = self.host._get_pcidev_info( + name, node_devs[name], net_devs, [], pci_devs) + expect_vf = { + "dev_id": "pci_0000_82_00_3", + "address": "0000:82:00.3", + "parent_addr": "0000:82:00.0", + "product_id": "101e", + "numa_node": 1, + "vendor_id": "15b3", + "label": "label_15b3_101e", + "dev_type": obj_fields.PciDeviceType.SRIOV_VF, + "capabilities": { + # Should be obtained from the parent PF in this case. + "vpd": {"card_serial_number": "MT2113X00000"}}, + } + self.assertEqual(expect_vf, actual_vf) + + # A VF with a VPD cap without a test parent dev (used to check the + # VPD code path when a VF's own VPD capability is used). + name = "pci_0001_82_00_3" + actual_vf = self.host._get_pcidev_info( + name, node_devs[name], net_devs, [], pci_devs) + expect_vf = { + "dev_id": "pci_0001_82_00_3", + "address": "0001:82:00.3", + "parent_addr": "0001:82:00.0", + "product_id": "101e", + "numa_node": 1, + "vendor_id": "15b3", + "label": "label_15b3_101e", + "dev_type": obj_fields.PciDeviceType.SRIOV_VF, + "capabilities": { + # Should be obtained from the parent PF in this case. + "vpd": {"card_serial_number": "MT2113XBEEF0"}}, + } + + # A VF without a VPD cap and without a test parent dev + # (used to check the code path where a VF VPD capability is + # checked but is not present and a parent PF info is not available). + name = "pci_0002_82_00_3" + actual_vf = self.host._get_pcidev_info( + name, node_devs[name], net_devs, [], pci_devs) + expect_vf = { + "dev_id": "pci_0002_82_00_3", + "address": "0002:82:00.3", + "parent_addr": "0002:82:00.0", + "product_id": "101e", + "numa_node": 1, + "vendor_id": "15b3", + "label": "label_15b3_101e", + "dev_type": obj_fields.PciDeviceType.SRIOV_VF, + } + + self.assertEqual(expect_vf, actual_vf) + def test_list_pci_devices(self): with mock.patch.object(self.host, "_list_devices") as mock_listDevices: self.host.list_pci_devices(8) diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 7129933f34..b1179b3d1f 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -3130,6 +3130,7 @@ class LibvirtConfigNodeDevice(LibvirtConfigObject): self.pci_capability = None self.mdev_information = None self.vdpa_capability = None + self.vpd_capability = None def parse_dom(self, xmldoc): super(LibvirtConfigNodeDevice, self).parse_dom(xmldoc) @@ -3183,6 +3184,7 @@ class LibvirtConfigNodeDevicePciCap(LibvirtConfigObject): self.numa_node = None self.fun_capability = [] self.mdev_capability = [] + self.vpd_capability = None self.interface = None self.address = None self.link_state = None @@ -3225,6 +3227,10 @@ class LibvirtConfigNodeDevicePciCap(LibvirtConfigObject): mdevcap = LibvirtConfigNodeDeviceMdevCapableSubFunctionCap() mdevcap.parse_dom(c) self.mdev_capability.append(mdevcap) + elif c.tag == "capability" and c.get('type') in ('vpd',): + vpdcap = LibvirtConfigNodeDeviceVpdCap() + vpdcap.parse_dom(c) + self.vpd_capability = vpdcap def pci_address(self): return "%04x:%02x:%02x.%01x" % ( @@ -3288,6 +3294,101 @@ class LibvirtConfigNodeDeviceMdevInformation(LibvirtConfigObject): self.iommu_group = int(c.get('number')) +class LibvirtConfigNodeDeviceVpdCap(LibvirtConfigObject): + + def __init__(self, **kwargs): + super().__init__( + root_name="capability", **kwargs) + self._card_name = None + self._change_level = None + self._manufacture_id = None + self._part_number = None + self._serial_number = None + self._asset_tag = None + self._ro_vendor_fields = {} + self._rw_vendor_fields = {} + self._rw_system_fields = {} + + @staticmethod + def _process_custom_field(fields_dict, field_element): + index = field_element.get('index') + if index: + fields_dict[index] = field_element.text + + def _parse_ro_fields(self, fields_element): + for e in fields_element: + if e.tag == 'change_level': + self._change_level = e.text + elif e.tag == 'manufacture_id': + self._manufacture_id = e.text + elif e.tag == 'part_number': + self._part_number = e.text + elif e.tag == 'serial_number': + self._serial_number = e.text + elif e.tag == 'vendor_field': + self._process_custom_field(self._ro_vendor_fields, e) + + def _parse_rw_fields(self, fields_element): + for e in fields_element: + if e.tag == 'asset_tag': + self._asset_tag = e.text + elif e.tag == 'vendor_field': + self._process_custom_field(self._rw_vendor_fields, e) + elif e.tag == 'system_field': + self._process_custom_field(self._rw_system_fields, e) + + def parse_dom(self, xmldoc): + super(LibvirtConfigNodeDeviceVpdCap, self).parse_dom(xmldoc) + for c in xmldoc: + if c.tag == "name": + self._card_name = c.text + if c.tag == "fields": + access = c.get('access') + if access: + if access == 'readonly': + self._parse_ro_fields(c) + elif access == 'readwrite': + self._parse_rw_fields(c) + else: + continue + + @property + def card_name(self): + return self._card_name + + @property + def change_level(self): + return self._change_level + + @property + def manufacture_id(self): + return self._manufacture_id + + @property + def part_number(self): + return self._part_number + + @property + def card_serial_number(self): + return self._serial_number + + @property + def asset_tag(self): + return self._asset_tag + + @property + def ro_vendor_fields(self): + return self._ro_vendor_fields + + @property + def rw_vendor_fields(self): + return self._rw_vendor_fields + + @property + def rw_system_fields(self): + return self._rw_system_fields + + class LibvirtConfigGuestRng(LibvirtConfigGuestDevice): def __init__(self, **kwargs): diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 2ea493d452..c8fd7e22ed 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -7742,9 +7742,15 @@ class LibvirtDriver(driver.ComputeDriver): vdpa_devs = [ dev for dev in devices.values() if "vdpa" in dev.listCaps() ] + pci_devs = { + name: dev for name, dev in devices.items() + if "pci" in dev.listCaps()} pci_info = [ - self._host._get_pcidev_info(name, dev, net_devs, vdpa_devs) - for name, dev in devices.items() if "pci" in dev.listCaps() + self._host._get_pcidev_info( + name, dev, net_devs, + vdpa_devs, list(pci_devs.values()) + ) + for name, dev in pci_devs.items() ] return jsonutils.dumps(pci_info) diff --git a/nova/virt/libvirt/host.py b/nova/virt/libvirt/host.py index 5c39dd320f..80663ab1dc 100644 --- a/nova/virt/libvirt/host.py +++ b/nova/virt/libvirt/host.py @@ -1229,12 +1229,51 @@ class Host(object): cfgdev.parse_str(xmlstr) return cfgdev.pci_capability.features + def _get_vf_parent_pci_vpd_info( + self, + vf_device: 'libvirt.virNodeDevice', + parent_pf_name: str, + candidate_devs: ty.List['libvirt.virNodeDevice'] + ) -> ty.Optional[vconfig.LibvirtConfigNodeDeviceVpdCap]: + """Returns PCI VPD info of a parent device of a PCI VF. + + :param vf_device: a VF device object to use for lookup. + :param str parent_pf_name: parent PF name formatted as pci_dddd_bb_ss_f + :param candidate_devs: devices that could be parent devs for the VF. + :returns: A VPD capability object of a parent device. + """ + parent_dev = next( + (dev for dev in candidate_devs if dev.name() == parent_pf_name), + None + ) + if parent_dev is None: + return None + + xmlstr = parent_dev.XMLDesc(0) + cfgdev = vconfig.LibvirtConfigNodeDevice() + cfgdev.parse_str(xmlstr) + return cfgdev.pci_capability.vpd_capability + + @staticmethod + def _get_vpd_card_serial_number( + dev: 'libvirt.virNodeDevice', + ) -> ty.Optional[ty.List[str]]: + """Returns a card serial number stored in PCI VPD (if present).""" + xmlstr = dev.XMLDesc(0) + cfgdev = vconfig.LibvirtConfigNodeDevice() + cfgdev.parse_str(xmlstr) + vpd_cap = cfgdev.pci_capability.vpd_capability + if not vpd_cap: + return None + return vpd_cap.card_serial_number + def _get_pcidev_info( self, devname: str, dev: 'libvirt.virNodeDevice', net_devs: ty.List['libvirt.virNodeDevice'], vdpa_devs: ty.List['libvirt.virNodeDevice'], + pci_devs: ty.List['libvirt.virNodeDevice'], ) -> ty.Dict[str, ty.Union[str, dict]]: """Returns a dict of PCI device.""" @@ -1314,6 +1353,52 @@ class Host(object): pcinet_info = self._get_pcinet_info(device, net_devs) if pcinet_info: return {'capabilities': {'network': pcinet_info}} + + return caps + + def _get_vpd_details( + device_dict: dict, + device: 'libvirt.virNodeDevice', + pci_devs: ty.List['libvirt.virNodeDevice'] + ) -> ty.Dict[str, ty.Dict[str, ty.Any]]: + """Get information from PCI VPD (if present). + + PCI/PCIe devices may include the optional VPD capability. It may + contain useful information such as the unique serial number + uniquely assigned at a factory. + + If a device is a VF and it does not contain the VPD capability, + a parent device's VPD is used (if present) as a fallback to + retrieve the unique add-in card number. Whether a VF exposes + the VPD capability or not may be controlled via a vendor-specific + firmware setting. + """ + caps: ty.Dict[str, ty.Dict[str, ty.Any]] = {} + # At the time of writing only the serial number had a clear + # use-case. However, the set of fields may be extended. + card_serial_number = self._get_vpd_card_serial_number(device) + + if (not card_serial_number and + device_dict.get('dev_type') == fields.PciDeviceType.SRIOV_VF + ): + # Format the address of a physical function to use underscores + # since that's how Libvirt formats the element content. + pf_addr = device_dict.get('parent_addr') + if not pf_addr: + LOG.warning("A VF device dict does not have a parent PF " + "address in it which is unexpected. Skipping " + "serial number retrieval") + return caps + + formatted_addr = pf_addr.replace('.', '_').replace(':', '_') + vpd_cap = self._get_vf_parent_pci_vpd_info( + device, f'pci_{formatted_addr}', pci_devs) + if vpd_cap is not None: + card_serial_number = vpd_cap.card_serial_number + + if card_serial_number: + caps = {'capabilities': { + 'vpd': {"card_serial_number": card_serial_number}}} return caps xmlstr = dev.XMLDesc(0) @@ -1340,6 +1425,7 @@ class Host(object): device.update( _get_device_type(cfgdev, address, dev, net_devs, vdpa_devs)) device.update(_get_device_capabilities(device, dev, net_devs)) + device.update(_get_vpd_details(device, dev, pci_devs)) return device def get_vdpa_nodedev_by_address( @@ -1361,7 +1447,7 @@ class Host(object): vdpa_devs = [ dev for dev in devices.values() if "vdpa" in dev.listCaps()] pci_info = [ - self._get_pcidev_info(name, dev, [], vdpa_devs) for name, dev + self._get_pcidev_info(name, dev, [], vdpa_devs, []) for name, dev in devices.items() if "pci" in dev.listCaps()] parent_dev = next( dev for dev in pci_info if dev['address'] == pci_address) diff --git a/releasenotes/notes/pci-vpd-capability-0d8039629db4afb8.yaml b/releasenotes/notes/pci-vpd-capability-0d8039629db4afb8.yaml new file mode 100644 index 0000000000..0ca3518351 --- /dev/null +++ b/releasenotes/notes/pci-vpd-capability-0d8039629db4afb8.yaml @@ -0,0 +1,20 @@ +--- +features: + - | + Add VPD capability parsing support when a PCI VPD capability is exposed + via node device XML in Libvirt. The XML data from Libvirt is parsed and + formatted into PCI device JSON dict that is sent to Nova API and is stored + in the extra_info column of a PciDevice. + + The code gracefully handles the lack of the capability since it is optional + or Libvirt may not support it in a particular release. + + A serial number is extracted from PCI VPD of network devices (if present) + and is sent to Neutron in port updates. + + Libvirt supports parsing the VPD capability from PCI/PCIe devices and + exposing it via nodedev XML as of 7.9.0. + + - https://libvirt.org/news.html#v7-9-0-2021-11-01 + - https://libvirt.org/drvnodedev.html#VPDCap +