From 9724ec118b880ca8713c69366f55df43494b8dbb Mon Sep 17 00:00:00 2001 From: Sahid Orentino Ferdjaoui Date: Tue, 17 Apr 2018 07:37:18 -0400 Subject: [PATCH] libvirt: place emulator threads on CONF.compute.cpu_shared_set Some workloads run best when the hypervisor overhead processes (emulator threads in libvirt/QEMU) can be placed on different physical host CPUs than other guest CPU resources. This allows those workloads to prevent latency spikes for guest vCPU threads. To ensure emulator threads are placed on a different set of physical CPUs than those running guest dedicated vCPUs, set the ``CONF.compute.cpu_shared_set`` configuration option to the set of host CPUs that should be used for best-effort CPU resources. Then set a flavor extra spec to ``hw:emulator_threads_policy=share`` to instruct nova to place that workload's emulator threads on that set of host CPUs. implement: bp/overhead-pin-set Signed-off-by: Sahid Orentino Ferdjaoui Change-Id: I0e63ab37d584ee3d7fde6553efaa61bfc866e67d --- doc/source/user/flavors.rst | 7 +- nova/tests/unit/virt/libvirt/test_driver.py | 113 +++++++++++++++++- nova/virt/libvirt/driver.py | 23 +++- ...lator-threads-policy-e5b57767104531b8.yaml | 15 +++ 4 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/emulator-threads-policy-e5b57767104531b8.yaml diff --git a/doc/source/user/flavors.rst b/doc/source/user/flavors.rst index 4d10a66beb..669266a409 100644 --- a/doc/source/user/flavors.rst +++ b/doc/source/user/flavors.rst @@ -559,8 +559,11 @@ Emulator threads policy Valid THREAD-POLICY values are: - - ``share``: (default) The emulator threads float across the pCPUs associated - to the guest. + - ``share``: (default) The emulator threads float across the pCPUs + associated to the guest. To place a workload's emulator threads on + a set of isolated physical CPUs, set ``share``` and + ``[compute]/cpu_shared_set`` configuration option to the set of + host CPUs that should be used for best-effort CPU resources. - ``isolate``: The emulator threads are isolated on a single pCPU. diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index c29d3ba350..bb5defff1e 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -3020,8 +3020,7 @@ class LibvirtConnTestCase(test.NoDBTestCase, # which are 6, 7 self.assertEqual(set([2, 3]), cfg.cputune.vcpusched[0].vcpus) - def test_get_guest_config_numa_host_instance_isolated_emulator_threads( - self): + def test_get_guest_config_numa_host_instance_isolated_emulthreads(self): instance_topology = objects.InstanceNUMATopology( emulator_threads_policy=( fields.CPUEmulatorThreadsPolicy.ISOLATE), @@ -3075,6 +3074,116 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertEqual(set([7]), cfg.cputune.vcpupin[2].cpuset) self.assertEqual(set([8]), cfg.cputune.vcpupin[3].cpuset) + def test_get_guest_config_numa_host_instance_shared_emulthreads_err( + self): + self.flags(cpu_shared_set="48-50", group="compute") + instance_topology = objects.InstanceNUMATopology( + emulator_threads_policy=( + fields.CPUEmulatorThreadsPolicy.SHARE), + cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={0: 4, 1: 5}, + cpuset_reserved=set([6])), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 3]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={2: 7, 3: 8})]) + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.numa_topology = instance_topology + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + + caps = vconfig.LibvirtConfigCaps() + caps.host = vconfig.LibvirtConfigCapsHost() + caps.host.cpu = vconfig.LibvirtConfigCPU() + caps.host.cpu.arch = "x86_64" + caps.host.topology = fakelibvirt.NUMATopology() + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, image_meta) + + with test.nested( + mock.patch.object( + objects.InstanceNUMATopology, "get_by_instance_uuid", + return_value=instance_topology), + mock.patch.object(host.Host, 'has_min_version', + return_value=True), + mock.patch.object(host.Host, "get_capabilities", + return_value=caps), + mock.patch.object( + hardware, 'get_vcpu_pin_set', + return_value=set([4, 5, 6, 7, 8])), + mock.patch.object(host.Host, 'get_online_cpus', + return_value=set(range(10))), + ): + # pCPUs [48-50] are not online + self.assertRaises(exception.Invalid, drvr._get_guest_config, + instance_ref, [], image_meta, disk_info) + + def test_get_guest_config_numa_host_instance_shared_emulator_threads( + self): + self.flags(cpu_shared_set="48-50", group="compute") + instance_topology = objects.InstanceNUMATopology( + emulator_threads_policy=( + fields.CPUEmulatorThreadsPolicy.SHARE), + cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={0: 4, 1: 5}, + cpuset_reserved=set([6])), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 3]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={2: 7, 3: 8})]) + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.numa_topology = instance_topology + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + + caps = vconfig.LibvirtConfigCaps() + caps.host = vconfig.LibvirtConfigCapsHost() + caps.host.cpu = vconfig.LibvirtConfigCPU() + caps.host.cpu.arch = "x86_64" + caps.host.topology = fakelibvirt.NUMATopology() + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, image_meta) + + with test.nested( + mock.patch.object( + objects.InstanceNUMATopology, "get_by_instance_uuid", + return_value=instance_topology), + mock.patch.object(host.Host, 'has_min_version', + return_value=True), + mock.patch.object(host.Host, "get_capabilities", + return_value=caps), + mock.patch.object( + hardware, 'get_vcpu_pin_set', + return_value=set([4, 5, 6, 7, 8])), + mock.patch.object(host.Host, 'get_online_cpus', + return_value=set(list(range(10)) + + [48, 50])), + ): + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + + # cpu_shared_set is configured with [48, 49, 50] but only + # [48, 50] are online. + self.assertEqual(set([48, 50]), cfg.cputune.emulatorpin.cpuset) + self.assertEqual(set([4]), cfg.cputune.vcpupin[0].cpuset) + self.assertEqual(set([5]), cfg.cputune.vcpupin[1].cpuset) + self.assertEqual(set([7]), cfg.cputune.vcpupin[2].cpuset) + self.assertEqual(set([8]), cfg.cputune.vcpupin[3].cpuset) + def test_get_cpu_numa_config_from_instance(self): topology = objects.InstanceNUMATopology(cells=[ objects.InstanceNUMACell(id=0, cpuset=set([1, 2]), memory=128), diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 804e328e02..104bda5a31 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -4290,20 +4290,37 @@ class LibvirtDriver(driver.ComputeDriver): a) If emulator threads policy is isolated, we pin emulator threads to one cpu we have reserved for it. - b) Otherwise; - b1) If realtime IS NOT enabled, the emulator threads are + b) If emulator threads policy is shared and CONF.cpu_shared_set is + defined, we pin emulator threads on the set of pCPUs defined by + CONF.cpu_shared_set + c) Otherwise; + c1) If realtime IS NOT enabled, the emulator threads are allowed to float cross all the pCPUs associated with the guest vCPUs. - b2) If realtime IS enabled, at least 1 vCPU is required + c2) If realtime IS enabled, at least 1 vCPU is required to be set aside for non-realtime usage. The emulator threads are allowed to float across the pCPUs that are associated with the non-realtime VCPUs. """ emulatorpin_cpuset = set([]) + shared_ids = hardware.get_cpu_shared_set() if emulator_threads_policy == fields.CPUEmulatorThreadsPolicy.ISOLATE: if object_numa_cell.cpuset_reserved: emulatorpin_cpuset = object_numa_cell.cpuset_reserved + elif ((emulator_threads_policy == + fields.CPUEmulatorThreadsPolicy.SHARE) and + shared_ids): + online_pcpus = self._host.get_online_cpus() + cpuset = shared_ids & online_pcpus + if not cpuset: + msg = (_("Invalid cpu_shared_set config, one or more of the " + "specified cpuset is not online. Online cpuset(s): " + "%(online)s, requested cpuset(s): %(req)s"), + {'online': sorted(online_pcpus), + 'req': sorted(shared_ids)}) + raise exception.Invalid(msg) + emulatorpin_cpuset = cpuset elif not wants_realtime or vcpu not in vcpus_rt: emulatorpin_cpuset = pin_cpuset.cpuset diff --git a/releasenotes/notes/emulator-threads-policy-e5b57767104531b8.yaml b/releasenotes/notes/emulator-threads-policy-e5b57767104531b8.yaml new file mode 100644 index 0000000000..f23ef611cc --- /dev/null +++ b/releasenotes/notes/emulator-threads-policy-e5b57767104531b8.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Introduces ``[compute]/cpu_shared_set`` option for compute nodes. + Some workloads run best when the hypervisor overhead processes + (emulator threads in libvirt/QEMU) can be placed on different + physical host CPUs than other guest CPU resources. This allows + those workloads to prevent latency spikes for guest vCPU threads. + + To place a workload's emulator threads on a set of isolated + physical CPUs, set the ``[compute]/cpu_shared_set`` configuration + option to the set of host CPUs that should be used for best-effort + CPU resources. Then set a flavor extra spec to + ``hw:emulator_threads_policy=share`` to instruct nova to place + that workload's emulator threads on that set of host CPUs. \ No newline at end of file