Add config option to configure TB cache size

Qemu>=5.0.0 bumped the default tb-cache size to 1GiB(from 32MiB)
and this made it difficult to run multiple guest VMs on systems
running with lower memory. With Libvirt>=8.0.0 it's possible to
configure lower tb-cache size.

Below config option is introduced to allow configure
TB cache size as per environment needs, this only
applies to 'virt_type=qemu':-

[libvirt]tb_cache_size

Also enable this flag in nova-next job.

[1] https://github.com/qemu/qemu/commit/600e17b26
[2] https://gitlab.com/libvirt/libvirt/-/commit/58bf03f85

Closes-Bug: #1949606
Implements: blueprint libvirt-tb-cache-size
Change-Id: I49d2276ff3d3cc5d560a1bd96f13408e798b256a

.zuul.yaml

@@ -362,6 +362,9 @@
               # machine type attaches more devices by default than pc
               num_pcie_ports: 24
               hw_machine_type: "x86_64=q35"
+              # Use lower TB cache than default(1GiB), only applicable with
+              # libvirt>=8.0.0
+              tb_cache_size: 128
             compute:
               # Switch off the provider association refresh, which should
               # reduce the number of placement calls in steady state. Added in

nova/conf/libvirt.py

@@ -118,6 +118,7 @@ Related options:
 * ``disk_prefix``: depends on this
 * ``cpu_mode``: depends on this
 * ``cpu_models``: depends on this
+* ``tb_cache_size``: depends on this
 """),
     cfg.StrOpt('connection_uri',
                default='',
@@ -900,6 +901,21 @@ Related options:
 
 * :oslo.config:option:`libvirt.device_detach_attempts`
 
+"""),
+    cfg.IntOpt('tb_cache_size',
+                min=0,
+                help="""
+Qemu>=5.0.0 bumped the default tb-cache size to 1GiB(from 32MiB) and this
+made it difficult to run multiple guest VMs on systems running with lower
+memory. With Libvirt>=8.0.0 this config option can be used to configure
+lower tb-cache size.
+
+Set it to > 0 to configure tb-cache for guest VMs.
+
+Related options:
+
+* ``compute_driver`` (libvirt)
+* ``virt_type`` (qemu)
 """),
 ]
 

nova/tests/unit/virt/libvirt/test_config.py

@@ -2414,6 +2414,14 @@ class LibvirtConfigGuestFeatureTest(LibvirtConfigBaseTest):
         xml = obj.to_xml()
         self.assertXmlEqual(xml, "<ioapic driver='libvirt'/>")
 
+    def test_feature_tcg(self):
+        obj = config.LibvirtConfigGuestFeatureTCG(10)
+        obj.driver = "libvirt"
+
+        xml = obj.to_xml()
+        self.assertXmlEqual(xml, '<tcg><tb-cache unit="MiB">10'
+                '</tb-cache></tcg>')
+
 
 class LibvirtConfigGuestTest(LibvirtConfigBaseTest):
 

nova/tests/unit/virt/libvirt/test_driver.py

@@ -21367,6 +21367,47 @@ class LibvirtConnTestCase(test.NoDBTestCase,
                     cpu_model = drvr._get_cpu_model_mapping(expect_model[0])
                     self.assertEqual(cpu_model, expect_model[0])
 
+    @mock.patch.object(fakelibvirt.Connection, 'getLibVersion',
+                       return_value=versionutils.convert_version_to_int(
+                           libvirt_driver.MIN_LIBVIRT_TB_CACHE_SIZE) - 1)
+    def test_supports_tb_cache_size_fail(self, mock_getversion):
+        self.flags(virt_type='qemu', group='libvirt')
+        self.flags(tb_cache_size=10, group='libvirt')
+        driver = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
+
+        self.assertRaises(exception.InvalidConfiguration,
+                          driver.init_host, 'dummyhost')
+
+    @mock.patch.object(libvirt_driver.LibvirtDriver,
+                       '_register_all_undefined_instance_details',
+                       new=mock.Mock())
+    @mock.patch.object(fakelibvirt.Connection, 'getLibVersion',
+                       return_value=versionutils.convert_version_to_int(
+                           libvirt_driver.MIN_LIBVIRT_TB_CACHE_SIZE))
+    def test_supports_tb_cache_size_ok(self, mock_getversion):
+        self.flags(virt_type='qemu', group='libvirt')
+        self.flags(tb_cache_size=10, group='libvirt')
+        driver = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
+        driver.init_host('dummyhost')
+
+    @mock.patch.object(fakelibvirt.Connection, 'getLibVersion',
+                       return_value=versionutils.convert_version_to_int(
+                            libvirt_driver.MIN_LIBVIRT_TB_CACHE_SIZE))
+    def test_get_guest_config_feature_tcg(self, mock_getversion):
+        self.flags(virt_type='qemu', group='libvirt')
+        self.flags(tb_cache_size=10, group='libvirt')
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        instance_ref = objects.Instance(**self.test_instance)
+        image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
+        disk_info = blockinfo.get_disk_info(
+            CONF.libvirt.virt_type, instance_ref, image_meta)
+
+        cfg = drvr._get_guest_config(
+            instance_ref, _fake_network_info(self), image_meta, disk_info)
+
+        expected = '<tcg><tb-cache unit="MiB">10</tb-cache></tcg>'
+        self.assertXmlEqual(expected, cfg.features[2].to_xml())
+
 
 class TestGuestConfigSysinfoSerialOS(test.NoDBTestCase):
     def setUp(self):

nova/virt/libvirt/config.py

@@ -2761,6 +2761,21 @@ class LibvirtConfigGuestFeatureSMM(LibvirtConfigGuestFeature):
         return root
 
 
+class LibvirtConfigGuestFeatureTCG(LibvirtConfigGuestFeature):
+
+    def __init__(self, cache_size, **kwargs):
+        super(LibvirtConfigGuestFeatureTCG, self).__init__("tcg", **kwargs)
+
+        self.cache_size = str(cache_size)
+
+    def format_dom(self):
+        root = super(LibvirtConfigGuestFeatureTCG, self).format_dom()
+        root.append(self._text_node("tb-cache", self.cache_size,
+                                    unit="MiB"))
+
+        return root
+
+
 class LibvirtConfigGuestFeaturePMU(LibvirtConfigGuestFeature):
 
     def __init__(self, state, **kwargs):

nova/virt/libvirt/driver.py

@@ -229,6 +229,8 @@ MIN_LIBVIRT_VIOMMU_VIRTIO_MODEL = (8, 3, 0)
 
 MIN_LIBVIRT_AARCH64_CPU_COMPARE = (6, 9, 0)
 
+MIN_LIBVIRT_TB_CACHE_SIZE = (8, 0, 0)
+
 # Virtuozzo driver support
 MIN_VIRTUOZZO_VERSION = (7, 0, 0)
 
@@ -743,6 +745,9 @@ class LibvirtDriver(driver.ComputeDriver):
 
         self._check_my_ip()
 
+        # TODO(ykarel) This can be dropped when MIN_LIBVIRT_VERSION>=8.0.0
+        self._supports_tb_cache_size()
+
         if (CONF.libvirt.virt_type == 'lxc' and
                 not (CONF.libvirt.uid_maps and CONF.libvirt.gid_maps)):
             LOG.warning("Running libvirt-lxc without user namespaces is "
@@ -1217,6 +1222,19 @@ class LibvirtDriver(driver.ComputeDriver):
                 "cpu_shared_set' and '[compute] cpu_dedicated_set', "
                 "respectively, and undefine 'vcpu_pin_set'.")
 
+    def _supports_tb_cache_size(self):
+        if (
+            CONF.libvirt.virt_type == 'qemu' and
+            CONF.libvirt.tb_cache_size and
+            CONF.libvirt.tb_cache_size > 0
+        ):
+            if not self._host.has_min_version(MIN_LIBVIRT_TB_CACHE_SIZE):
+                raise exception.InvalidConfiguration(
+                    _("Nova requires libvirt version %s or greater "
+                      "with '[libvirt] tb_cache_size' "
+                      "configured.") %
+                    libvirt_utils.version_to_string(MIN_LIBVIRT_TB_CACHE_SIZE))
+
     def _prepare_migration_flags(self):
         migration_flags = 0
 
@@ -6176,6 +6194,14 @@ class LibvirtDriver(driver.ComputeDriver):
             if not CONF.workarounds.libvirt_disable_apic:
                 guest.add_feature(vconfig.LibvirtConfigGuestFeatureAPIC())
 
+        if (
+            CONF.libvirt.virt_type == 'qemu' and
+            CONF.libvirt.tb_cache_size and
+            CONF.libvirt.tb_cache_size > 0
+        ):
+            guest.add_feature(vconfig.LibvirtConfigGuestFeatureTCG(
+                CONF.libvirt.tb_cache_size))
+
         if CONF.libvirt.virt_type in ('qemu', 'kvm') and os_type == 'windows':
             hv = vconfig.LibvirtConfigGuestFeatureHyperV()
             hv.relaxed = True

releasenotes/notes/libvirt-tb-cache-size-c0d0d672c0950393.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    Qemu>=5.0.0 bumped the default tb-cache size to 1GiB(from 32MiB) and this
+    made it difficult to run multiple guest VMs on systems running with lower
+    memory. With Libvirt>=8.0.0 it's possible to configure lower tb-cache
+    size. A new config option is introduced:
+
+    ``[libvirt]tb_cache_size``
+
+    This config option can be used to configure tb_cache size
+    for guest VMs, it's only applicable with ``virt_type=qemu``.