diff --git a/doc/source/admin/hw-emulation-architecture.rst b/doc/source/admin/hw-emulation-architecture.rst new file mode 100644 index 0000000000..71222fa043 --- /dev/null +++ b/doc/source/admin/hw-emulation-architecture.rst @@ -0,0 +1,133 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +============================================================================ +hw_emulation_architecture - Configuring QEMU instance emulation architecture +============================================================================ + +.. versionadded:: 25.0.0 (Yoga) + + The libvirt driver now allows for handling of specific cpu architectures + when defined within the image metadata properties, to be emulated through + QEMU. + + Added ``hw_emulation_architecture`` as an available image_meta property. + +.. note:: + + The following only applies to environments using libvirt compute hosts. + and should be considered experimental in its entirety, during its first + release as a feature. + +Introduction +------------ + +This capability is to fill a need with environments that do not have the +capability to support the various cpu architectures that are present today +with physical hardware. A small subset of architectures that are supported +both within libvirt and QEMU have been selected as prime candidates for +emulation support. + +While support has been added for the below base architectures, this does +not guarantee that every subset or custom operating system that leverages +one of these architectures will function. + +Configure +--------- + +------------------- +QEMU Binary Support +------------------- + +To ensure that libvirt and QEMU can properly handle the level of cpu +emulation desired by the end-user, you are required to install the specific +``qemu-system-XXX``, ``qemu-efi-arm``, ``qemu-efi-aarch64`` binaries on the +compute nodes that will be providing support. + +--------------- +Console Support +--------------- + +Consideration need to be made in regards to which architectures you want to +support, as there are limitations on support through spice, novnc, and +serial. All testing and validation has been done to ensure that spice and +serial connections function as expected. + +- ``AARCH64`` - Spice & Serial +- ``S390X`` - Serial +- ``PPC64LE`` - Spice & Serial +- ``MIPSEL`` - untested + +-------------------------------- +Supported Emulated Architectures +-------------------------------- + +The supported emulated architectures require specific image meta +properties to be set in order to trigger the proper settings to be +configured by libvirtd. + +For end users the emulation architecture of an instance is controlled by the +selection of an image with the ``hw_emulation_architecture`` image metadata +property set. + + +AARCH64 +~~~~~~~ + +``Tested and Validated as functional`` + +.. code-block:: shell + + $ openstack image set --property hw_emulation_architecture=aarch64 $IMAGE + $ openstack image set --property hw_machine_type=virt $IMAGE + $ openstack image set --property hw_firmware_type=uefi $IMAGE + +S390x +~~~~~ + +``Tested and Validated as functional`` + +.. code-block:: shell + + $ openstack image set --property hw_emulation_architecture=s390x $IMAGE + $ openstack image set --property hw_machine_type=s390-ccw-virtio $IMAGE + $ openstack image set --property hw_video_model=virtio $IMAGE + +PPC64LE +~~~~~~~ + +``Tested and Validated as functional`` + +.. code-block:: shell + + $ openstack image set --property hw_emulation_architecture=ppc64le $IMAGE + $ openstack image set --property hw_machine_type=pseries $IMAGE + + +MIPSEL +~~~~~~ + +``Testing and validation is ongoing to overcome PCI issues`` + +.. note:: + + Support is currently impacted, one current method for support is manually + patching and compiling as defined in libvirt bug + `XML error: No PCI buses available`_. + +.. _`XML error: No PCI buses available`: https://bugzilla.redhat.com/show_bug.cgi?id=1432101 + +.. code-block:: shell + + $ openstack image set --property hw_emulation_architecture=mipsel $IMAGE + $ openstack image set --property hw_machine_type=virt $IMAGE diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 04078f0852..e83f680df2 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -227,3 +227,4 @@ Once you are running nova, the following information is extremely useful. upgrades node-down hw-machine-type + hw-emulation-architecture diff --git a/nova/scheduler/request_filter.py b/nova/scheduler/request_filter.py index be1455df88..bd237b06ca 100644 --- a/nova/scheduler/request_filter.py +++ b/nova/scheduler/request_filter.py @@ -212,6 +212,8 @@ def transform_image_metadata(ctxt, request_spec): 'hw_disk_bus': 'COMPUTE_STORAGE_BUS', 'hw_video_model': 'COMPUTE_GRAPHICS_MODEL', 'hw_vif_model': 'COMPUTE_NET_VIF_MODEL', + 'hw_architecture': 'HW_ARCH', + 'hw_emulation_architecture': 'COMPUTE_ARCH', } trait_names = [] diff --git a/nova/tests/functional/libvirt/test_uefi.py b/nova/tests/functional/libvirt/test_uefi.py index 1eee1ab5e1..40becf425e 100644 --- a/nova/tests/functional/libvirt/test_uefi.py +++ b/nova/tests/functional/libvirt/test_uefi.py @@ -14,6 +14,7 @@ # under the License. import datetime +import re from lxml import etree from oslo_log import log as logging @@ -47,6 +48,8 @@ class UEFIServersTest(base.ServersTestBase): orig_create = nova.virt.libvirt.guest.Guest.create def fake_create(cls, xml, host): + xml = re.sub('type arch.*machine', + 'type machine', xml) tree = etree.fromstring(xml) self.assertXmlEqual( """ diff --git a/nova/tests/unit/scheduler/test_request_filter.py b/nova/tests/unit/scheduler/test_request_filter.py index 7be7f8341d..57f2f93bf6 100644 --- a/nova/tests/unit/scheduler/test_request_filter.py +++ b/nova/tests/unit/scheduler/test_request_filter.py @@ -406,13 +406,15 @@ class TestRequestFilter(test.NoDBTestCase): self.assertIn('took %.1f seconds', log_lines[1]) @mock.patch.object(request_filter, 'LOG', new=mock.Mock()) - def test_transform_image_metadata(self): + def test_transform_image_metadata_x86(self): self.flags(image_metadata_prefilter=True, group='scheduler') properties = objects.ImageMetaProps( hw_disk_bus=objects.fields.DiskBus.SATA, hw_cdrom_bus=objects.fields.DiskBus.IDE, hw_video_model=objects.fields.VideoModel.QXL, - hw_vif_model=network_model.VIF_MODEL_VIRTIO + hw_vif_model=network_model.VIF_MODEL_VIRTIO, + hw_architecture=objects.fields.Architecture.X86_64, + hw_emulation_architecture=objects.fields.Architecture.AARCH64 ) reqspec = objects.RequestSpec( image=objects.ImageMeta(properties=properties), @@ -426,6 +428,36 @@ class TestRequestFilter(test.NoDBTestCase): 'COMPUTE_NET_VIF_MODEL_VIRTIO', 'COMPUTE_STORAGE_BUS_IDE', 'COMPUTE_STORAGE_BUS_SATA', + 'HW_ARCH_X86_64', + 'COMPUTE_ARCH_AARCH64', + } + self.assertEqual(expected, reqspec.root_required) + + @mock.patch.object(request_filter, 'LOG', new=mock.Mock()) + def test_transform_image_metadata_aarch64(self): + self.flags(image_metadata_prefilter=True, group='scheduler') + properties = objects.ImageMetaProps( + hw_disk_bus=objects.fields.DiskBus.SATA, + hw_cdrom_bus=objects.fields.DiskBus.IDE, + hw_video_model=objects.fields.VideoModel.QXL, + hw_vif_model=network_model.VIF_MODEL_VIRTIO, + hw_architecture=objects.fields.Architecture.AARCH64, + hw_emulation_architecture=objects.fields.Architecture.X86_64 + ) + reqspec = objects.RequestSpec( + image=objects.ImageMeta(properties=properties), + flavor=objects.Flavor(extra_specs={}), + ) + self.assertTrue( + request_filter.transform_image_metadata(None, reqspec) + ) + expected = { + 'COMPUTE_GRAPHICS_MODEL_QXL', + 'COMPUTE_NET_VIF_MODEL_VIRTIO', + 'COMPUTE_STORAGE_BUS_IDE', + 'COMPUTE_STORAGE_BUS_SATA', + 'HW_ARCH_AARCH64', + 'COMPUTE_ARCH_X86_64', } self.assertEqual(expected, reqspec.root_required) diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 371a627744..bdcb83deb2 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -5218,7 +5218,8 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.mock_uname.return_value = fakelibvirt.os_uname( 'Linux', '', '5.4.0-0-generic', '', fields.Architecture.AARCH64) drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) - self.assertTrue(drvr._check_uefi_support(None)) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + self.assertTrue(drvr._check_uefi_support(image_meta)) def test_get_guest_config_with_block_device(self): drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) @@ -5769,6 +5770,7 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.flags(enabled=True, group='serial_console') drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) instance = objects.Instance(**self.test_instance) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) expected = { fields.Architecture.X86_64: vconfig.LibvirtConfigGuestSerial, @@ -5781,7 +5783,10 @@ class LibvirtConnTestCase(test.NoDBTestCase, guest = vconfig.LibvirtConfigGuest() drvr._create_consoles( - guest_cfg=guest, instance=instance, flavor={}, image_meta={}) + guest_cfg=guest, + instance=instance, + flavor={}, + image_meta=image_meta) self.assertEqual(1, len(guest.devices)) console_device = guest.devices[0] self.assertIsInstance(console_device, device_type) @@ -5993,9 +5998,13 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.flags(enabled=serial_enabled, group='serial_console') guest_cfg = vconfig.LibvirtConfigGuest() instance = objects.Instance(**self.test_instance) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) drvr._create_consoles( - guest_cfg, instance=instance, flavor=None, image_meta=None) + guest_cfg, + instance=instance, + flavor=None, + image_meta=image_meta) self.assertEqual(1, len(guest_cfg.devices)) device = guest_cfg.devices[0] @@ -7975,6 +7984,33 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertIsInstance(conf.cpu, vconfig.LibvirtConfigGuestCPU) self.assertEqual(conf.cpu.mode, 'custom') + def test_get_x86_64_hw_emulated_architecture_aarch64(self): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + 'properties': { + 'hw_architecture': 'x86_64', + 'hw_emulation_architecture': 'aarch64', + 'hw_machine_type': 'virt', + 'hw_firmware_type': 'uefi', + }}) + + self.assertEqual(drvr._check_emulation_arch(image_meta), + 'aarch64') + + def test_get_x86_64_hw_emulated_architecture_ppc64(self): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + image_meta = objects.ImageMeta.from_dict({ + "disk_format": "raw", + 'properties': { + 'hw_architecture': 'x86_64', + 'hw_emulation_architecture': 'ppc64le', + 'hw_machine_type': 'pseries', + }}) + + self.assertEqual(drvr._check_emulation_arch(image_meta), + 'ppc64le') + @mock.patch.object(libvirt_driver.LOG, 'warning') def test_get_guest_cpu_config_custom_with_extra_flags(self, mock_warn): diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 2cce4a49eb..1a81be3ade 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -2823,6 +2823,7 @@ class LibvirtConfigGuest(LibvirtConfigObject): self.os_init_path = None self.os_boot_dev = [] self.os_smbios = None + self.os_arch = None self.os_mach_type = None self.os_bootmenu = False self.devices = [] @@ -2865,6 +2866,8 @@ class LibvirtConfigGuest(LibvirtConfigObject): os.set("firmware", self.os_firmware) type_node = self._text_node("type", self.os_type) + if self.os_arch is not None: + type_node.set("arch", self.os_arch) if self.os_mach_type is not None: type_node.set("machine", self.os_mach_type) os.append(type_node) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 0c9d85f69b..476bec4c82 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -5118,6 +5118,43 @@ class LibvirtDriver(driver.ComputeDriver): else: mount.get_manager().host_down() + def _check_emulation_arch(self, image_meta): + # NOTE(chateaulav) In order to support emulation via qemu, + # there are required metadata properties that need applied + # to the designated glance image. The config drive is not + # supported. This leverages the hw_architecture and + # hw_emulation_architecture image_meta fields to allow for + # emulation to take advantage of all physical multiarch work + # being done. + # + # aarch64 emulation support metadata values: + # 'hw_emulation_architecture=aarch64' + # 'hw_firmware_type=uefi' + # 'hw_machine_type=virt' + # + # ppc64le emulation support metadata values: + # 'hw_emulation_architecture=ppc64le' + # 'hw_machine_type=pseries' + # + # s390x emulation support metadata values: + # 'hw_emulation_architecture=s390x' + # 'hw_machine_type=s390-ccw-virtio' + # 'hw_video_model=virtio' + # + # TODO(chateaulav) Further Work to be done: + # testing mips functionality while waiting on redhat libvirt + # patch https://listman.redhat.com/archives/libvir-list/ + # 2016-May/msg00197.html + # + # https://bugzilla.redhat.com/show_bug.cgi?id=1432101 + emulation_arch = image_meta.properties.get("hw_emulation_architecture") + if emulation_arch: + arch = emulation_arch + else: + arch = libvirt_utils.get_arch(image_meta) + + return arch + def _get_cpu_model_mapping(self, model): """Get the CPU model mapping @@ -5258,7 +5295,7 @@ class LibvirtDriver(driver.ComputeDriver): def _get_guest_cpu_config(self, flavor, image_meta, guest_cpu_numa_config, instance_numa_topology): - arch = libvirt_utils.get_arch(image_meta) + arch = self._check_emulation_arch(image_meta) cpu = self._get_guest_cpu_model_config(flavor, arch) if cpu is None: @@ -5271,6 +5308,23 @@ class LibvirtDriver(driver.ComputeDriver): cpu.threads = topology.threads cpu.numa = guest_cpu_numa_config + caps = self._host.get_capabilities() + if arch != caps.host.cpu.arch: + # Try emulating. Other arch configs will go here + cpu.mode = None + if arch == fields.Architecture.AARCH64: + cpu.model = "cortex-a57" + elif arch == fields.Architecture.PPC64LE: + cpu.model = "POWER8" + # TODO(chateaulav): re-evaluate when libvirtd adds overall + # RISCV suuport as a supported architecture, as there is no + # cpu models associated, this simply associates X vcpus to the + # guest according to the flavor. Thes same issue should be + # present with mipsel due to same limitation, but has not been + # tested. + elif arch == fields.Architecture.MIPSEL: + cpu = None + return cpu def _get_guest_disk_config( @@ -5957,7 +6011,7 @@ class LibvirtDriver(driver.ComputeDriver): clk.add_timer(tmrtc) hpet = image_meta.properties.get('hw_time_hpet', False) - guestarch = libvirt_utils.get_arch(image_meta) + guestarch = self._check_emulation_arch(image_meta) if guestarch in (fields.Architecture.I686, fields.Architecture.X86_64): # NOTE(rfolco): HPET is a hardware timer for x86 arch. @@ -6020,7 +6074,7 @@ class LibvirtDriver(driver.ComputeDriver): if CONF.libvirt.virt_type in ("qemu", "kvm"): # vmcoreinfo support is x86, ARM-only for now - guestarch = libvirt_utils.get_arch(image_meta) + guestarch = self._check_emulation_arch(image_meta) if guestarch in ( fields.Architecture.I686, fields.Architecture.X86_64, fields.Architecture.AARCH64, @@ -6081,8 +6135,7 @@ class LibvirtDriver(driver.ComputeDriver): raise exception.InvalidVideoMode(model=video_type) return video_type - guestarch = libvirt_utils.get_arch(image_meta) - + guestarch = self._check_emulation_arch(image_meta) if CONF.libvirt.virt_type == 'parallels': return 'vga' @@ -6114,8 +6167,9 @@ class LibvirtDriver(driver.ComputeDriver): # NOTE(kevinz): Only virtio device type is supported by AARCH64 # so use 'virtio' instead when running on AArch64 hardware. return 'virtio' - - if CONF.spice.enabled: + elif guestarch == fields.Architecture.MIPSEL: + return 'virtio' + elif CONF.spice.enabled: return 'qxl' # NOTE(lyarwood): Return None and default to the default of @@ -6355,7 +6409,14 @@ class LibvirtDriver(driver.ComputeDriver): flavor: 'objects.Flavor', ) -> None: if CONF.libvirt.virt_type in ("kvm", "qemu"): - arch = libvirt_utils.get_arch(image_meta) + caps = self._host.get_capabilities() + host_arch = caps.host.cpu.arch + arch = self._check_emulation_arch(image_meta) + guest.os_arch = self._check_emulation_arch(image_meta) + if arch != host_arch: + # If emulating, downgrade to qemu + guest.virt_type = "qemu" + if arch in (fields.Architecture.I686, fields.Architecture.X86_64): guest.sysinfo = self._get_guest_config_sysinfo(instance) guest.os_smbios = vconfig.LibvirtConfigGuestSMBIOS() @@ -6502,13 +6563,17 @@ class LibvirtDriver(driver.ComputeDriver): self._create_consoles_qemu_kvm( guest_cfg, instance, flavor, image_meta) + def _is_mipsel_guest(self, image_meta): + archs = (fields.Architecture.MIPSEL, fields.Architecture.MIPS64EL) + return self._check_emulation_arch(image_meta) in archs + def _is_s390x_guest(self, image_meta): - s390x_archs = (fields.Architecture.S390, fields.Architecture.S390X) - return libvirt_utils.get_arch(image_meta) in s390x_archs + archs = (fields.Architecture.S390, fields.Architecture.S390X) + return self._check_emulation_arch(image_meta) in archs def _is_ppc64_guest(self, image_meta): archs = (fields.Architecture.PPC64, fields.Architecture.PPC64LE) - return libvirt_utils.get_arch(image_meta) in archs + return self._check_emulation_arch(image_meta) in archs def _create_consoles_qemu_kvm(self, guest_cfg, instance, flavor, image_meta): @@ -6677,7 +6742,19 @@ class LibvirtDriver(driver.ComputeDriver): # controller (x86 gets one by default) usbhost.model = None if not self._guest_needs_usb(guest, image_meta): - usbhost.model = 'none' + archs = ( + fields.Architecture.PPC, + fields.Architecture.PPC64, + fields.Architecture.PPC64LE, + ) + if self._check_emulation_arch(image_meta) in archs: + # NOTE(chateaulav): during actual testing and implementation + # it wanted None for ppc, as this removes it from the domain + # xml, where 'none' adds it but then disables it causing + # libvirt errors and the instances not being able to build + usbhost.model = None + else: + usbhost.model = 'none' guest.add_device(usbhost) def _guest_add_pcie_root_ports(self, guest): @@ -7174,7 +7251,7 @@ class LibvirtDriver(driver.ComputeDriver): # libvirt will automatically add a PS2 keyboard) # TODO(stephenfin): We might want to do this for other non-x86 # architectures - arch = libvirt_utils.get_arch(image_meta) + arch = self._check_emulation_arch(image_meta) if arch != fields.Architecture.AARCH64: return None diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py index da2a6e8b8a..834f242c79 100644 --- a/nova/virt/libvirt/utils.py +++ b/nova/virt/libvirt/utils.py @@ -526,6 +526,9 @@ def get_cpu_model_from_arch(arch: str) -> str: mode = 'qemu32' elif arch == obj_fields.Architecture.PPC64LE: mode = 'POWER8' + # TODO(chateaulav): Testing of emulated archs ongoing + # elif arch == obj_fields.Architecture.MIPSEL: + # mode = '24Kf-mips-cpu' # NOTE(kevinz): In aarch64, cpu model 'max' will offer the capabilities # that all the stuff it can currently emulate, both for "TCG" and "KVM" elif arch == obj_fields.Architecture.AARCH64: @@ -568,6 +571,7 @@ def get_default_machine_type(arch: str) -> ty.Optional[str]: default_mtypes = { obj_fields.Architecture.ARMV7: "virt", obj_fields.Architecture.AARCH64: "virt", + obj_fields.Architecture.PPC64LE: "pseries", obj_fields.Architecture.S390: "s390-ccw-virtio", obj_fields.Architecture.S390X: "s390-ccw-virtio", obj_fields.Architecture.I686: "pc", diff --git a/releasenotes/notes/bp-pick-guest-arch-based-on-host-arch-in-libvirt-driver-f087c3799d388bb6.yaml b/releasenotes/notes/bp-pick-guest-arch-based-on-host-arch-in-libvirt-driver-f087c3799d388bb6.yaml new file mode 100644 index 0000000000..d10f753180 --- /dev/null +++ b/releasenotes/notes/bp-pick-guest-arch-based-on-host-arch-in-libvirt-driver-f087c3799d388bb6.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + image meta now includes the ``hw_emulation_architecture`` property. + This allows an operator to define their emulated cpu architecture for + an image, and nova will deploy accordingly. + + See the `spec`_ for more details and reasoning. + + .. _spec: https://specs.openstack.org/openstack/nova-specs/specs/yoga/approved/pick-guest-arch-based-on-host-arch-in-libvirt-driver.html \ No newline at end of file