Consolidate [image_cache] conf options

Blueprint image-precache-support added a conf section called
[image_cache], so it makes sense to move all the existing image
cache-related conf options into it.

Old:
[DEFAULT]image_cache_manager_interval
[DEFAULT]image_cache_subdirectory_name
[DEFAULT]remove_unused_base_images
[DEFAULT]remove_unused_original_minimum_age_seconds
[libvirt]remove_unused_resized_minimum_age_seconds

New:
[image_cache]manager_interval
[image_cache]subdirectory_name
[image_cache]remove_unused_base_images
[image_cache]remove_unused_original_minimum_age_seconds
[image_cache]remove_unused_resized_minimum_age_seconds

Change-Id: I3c49825ac0d70152b6c8ee4c8ca01546265f4b80
Partial-Bug: #1847302

doc/source/admin/configuration/hypervisor-vmware.rst

@@ -926,18 +926,11 @@ cached images are stored.
    have a shared file system.
 
 You can automatically purge unused images after a specified period of time.  To
-configure this action, set these options in the ``DEFAULT`` section in the
+configure this action, set these options in the :oslo.config:group`image_cache`
-``nova.conf`` file:
+section in the ``nova.conf`` file:
 
-``remove_unused_base_images``
+* :oslo.config:option:`image_cache.remove_unused_base_images`
-  Set this option to ``True`` to specify that unused images should be removed
+* :oslo.config:option:`image_cache.remove_unused_original_minimum_age_seconds`
-  after the duration specified in the
-  ``remove_unused_original_minimum_age_seconds`` option.  The default is
-  ``True``.
-
-``remove_unused_original_minimum_age_seconds``
-  Specifies the duration in seconds after which an unused image is purged from
-  the cache. The default is ``86400`` (24 hours).
 
 .. _vmware-networking:
 

doc/source/admin/image-caching.rst

@@ -38,13 +38,8 @@ default). Those unused images are deleted from the cache directory
 until they are needed again.
 
 For more information about configuring image cache behavior, see the
-documentation for the following configuration options:
+documentation for the configuration options in the
-
+:oslo.config:group:`image_cache` group.
-- :oslo.config:option:`image_cache_subdirectory_name`
-- :oslo.config:option:`image_cache_manager_interval`
-- :oslo.config:option:`remove_unused_base_images`
-- :oslo.config:option:`remove_unused_original_minimum_age_seconds`
-- :oslo.config:option:`image_cache.precache_concurrency`
 
 .. note::
 

nova/compute/manager.py

@@ -9698,7 +9698,7 @@ class ComputeManager(manager.Manager):
             else:
                 self._process_instance_event(instance, event)
 
-    @periodic_task.periodic_task(spacing=CONF.image_cache_manager_interval,
+    @periodic_task.periodic_task(spacing=CONF.image_cache.manager_interval,
                                  external_process_ok=True)
     def _run_image_cache_manager_pass(self, context):
         """Run a single pass of the image cache manager."""

nova/conf/compute.py

@@ -264,22 +264,6 @@ driver (for qcow and raw), or loop (for raw).
         default=10,
         min=0,
         help='Amount of time, in seconds, to wait for NBD device start up.'),
-    cfg.StrOpt('image_cache_subdirectory_name',
-        default='_base',
-        help="""
-Location of cached images.
-
-This is NOT the full path - just a folder name relative to '$instances_path'.
-For per-compute-host cached images, set to '_base_$my_ip'
-"""),
-    cfg.BoolOpt('remove_unused_base_images',
-        default=True,
-        help='Should unused base images be removed?'),
-    cfg.IntOpt('remove_unused_original_minimum_age_seconds',
-        default=(24 * 3600),
-        help="""
-Unused unresized base images younger than this will not be removed.
-"""),
     cfg.StrOpt('pointer_model',
         default='usbtablet',
         choices=[
@@ -954,31 +938,6 @@ Possible values:
 ]
 
 interval_opts = [
-    cfg.IntOpt('image_cache_manager_interval',
-        default=2400,
-        min=-1,
-        help="""
-Number of seconds to wait between runs of the image cache manager.
-
-Note that when using shared storage for the ``[DEFAULT]/instances_path``
-configuration option across multiple nova-compute services, this periodic
-could process a large number of instances. Similarly, using a compute driver
-that manages a cluster (like vmwareapi.VMwareVCDriver) could result in
-processing a large number of instances. Therefore you may need to adjust the
-time interval for the anticipated load, or only run on one nova-compute
-service within a shared storage aggregate.
-
-Possible values:
-
-* 0: run at the default interval of 60 seconds (not recommended)
-* -1: disable
-* Any other value
-
-Related options:
-
-* ``[DEFAULT]/compute_driver``
-* ``[DEFAULT]/instances_path``
-"""),
     cfg.IntOpt('bandwidth_poll_interval',
         default=600,
         help="""

nova/conf/imagecache.py

@@ -19,6 +19,59 @@ imagecache_group = cfg.OptGroup(
 A collection of options specific to image caching.
 """)
 imagecache_opts = [
+    cfg.IntOpt('manager_interval',
+        default=2400,
+        min=-1,
+        deprecated_name='image_cache_manager_interval',
+        deprecated_group='DEFAULT',
+        help="""
+Number of seconds to wait between runs of the image cache manager.
+
+Note that when using shared storage for the ``[DEFAULT]/instances_path``
+configuration option across multiple nova-compute services, this periodic
+could process a large number of instances. Similarly, using a compute driver
+that manages a cluster (like vmwareapi.VMwareVCDriver) could result in
+processing a large number of instances. Therefore you may need to adjust the
+time interval for the anticipated load, or only run on one nova-compute
+service within a shared storage aggregate.
+
+Possible values:
+
+* 0: run at the default interval of 60 seconds (not recommended)
+* -1: disable
+* Any other value
+
+Related options:
+
+* ``[DEFAULT]/compute_driver``
+* ``[DEFAULT]/instances_path``
+"""),
+    cfg.StrOpt('subdirectory_name',
+        default='_base',
+        deprecated_name='image_cache_subdirectory_name',
+        deprecated_group='DEFAULT',
+        help="""
+Location of cached images.
+
+This is NOT the full path - just a folder name relative to '$instances_path'.
+For per-compute-host cached images, set to '_base_$my_ip'
+"""),
+    cfg.BoolOpt('remove_unused_base_images',
+        default=True,
+        deprecated_group='DEFAULT',
+        help='Should unused base images be removed?'),
+    cfg.IntOpt('remove_unused_original_minimum_age_seconds',
+        default=(24 * 3600),
+        deprecated_group='DEFAULT',
+        help="""
+Unused unresized base images younger than this will not be removed.
+"""),
+    cfg.IntOpt('remove_unused_resized_minimum_age_seconds',
+        default=3600,
+        deprecated_group='libvirt',
+        help="""
+Unused resized base images younger than this will not be removed.
+"""),
     cfg.IntOpt('precache_concurrency',
                default=1,
                min=1,

nova/conf/libvirt.py

@@ -953,13 +953,6 @@ Requires:
 """),
 ]
 
-libvirt_imagecache_opts = [
-    cfg.IntOpt('remove_unused_resized_minimum_age_seconds',
-               default=3600,
-               help='Unused resized base images younger than this will not be '
-                    'removed'),
-]
-
 libvirt_lvm_opts = [
     cfg.StrOpt('volume_clear',
         default='zero',
@@ -1364,7 +1357,6 @@ For example::
 ALL_OPTS = list(itertools.chain(
     libvirt_general_opts,
     libvirt_imagebackend_opts,
-    libvirt_imagecache_opts,
     libvirt_lvm_opts,
     libvirt_utils_opts,
     libvirt_vif_opts,

nova/tests/unit/virt/hyperv/test_imagecache.py

@@ -188,7 +188,8 @@ class ImageCacheTestCase(test_base.HyperVBaseTestCase):
 
     @ddt.data(True, False)
     def test_age_and_verify_cached_images(self, remove_unused_base_images):
-        self.flags(remove_unused_base_images=remove_unused_base_images)
+        self.flags(remove_unused_base_images=remove_unused_base_images,
+                   group='image_cache')
 
         fake_images = [mock.sentinel.FAKE_IMG1, mock.sentinel.FAKE_IMG2]
         fake_used_images = [mock.sentinel.FAKE_IMG1]

nova/tests/unit/virt/libvirt/test_driver.py

@@ -739,7 +739,7 @@ class CacheConcurrencyTestCase(test.NoDBTestCase):
 
         def fake_exists(fname):
             basedir = os.path.join(CONF.instances_path,
-                                   CONF.image_cache_subdirectory_name)
+                                   CONF.image_cache.subdirectory_name)
             if fname == basedir or fname == self.lock_path:
                 return True
             return False
@@ -12736,7 +12736,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
         conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
 
         base_dir = os.path.join(CONF.instances_path,
-                                CONF.image_cache_subdirectory_name)
+                                CONF.image_cache.subdirectory_name)
         trusted_certs = objects.TrustedCerts(
             ids=['0b5d2c72-12cc-4ba6-a8d7-3ff5cc1d8cb8',
                  '674736e3-f25c-405c-8362-bbf991e0ce0a'])
@@ -12847,7 +12847,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
         drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
 
         base_dir = os.path.join(CONF.instances_path,
-                                CONF.image_cache_subdirectory_name)
+                                CONF.image_cache.subdirectory_name)
         instance = objects.Instance(**self.test_instance)
         disk_info_byname = fake_disk_info_byname(instance)
 
@@ -23578,7 +23578,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
         # are fully mocked out and are just testing string formatting in this
         # test.
         self.flags(instances_path='/nova/instances')
-        self.flags(image_cache_subdirectory_name='cache')
+        self.flags(subdirectory_name='cache', group='image_cache')
         expected_fn = os.path.join('/nova/instances/cache',
                                    imagecache.get_cache_fname('an-image'))
 
@@ -23612,7 +23612,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
         # are fully mocked out and are just testing string formatting in this
         # test.
         self.flags(instances_path='/nova/instances')
-        self.flags(image_cache_subdirectory_name='cache')
+        self.flags(subdirectory_name='cache', group='image_cache')
         expected_fn = os.path.join('/nova/instances/cache',
                                    imagecache.get_cache_fname('an-image'))
 

nova/tests/unit/virt/libvirt/test_imagebackend.py

@@ -306,7 +306,7 @@ class FlatTestCase(_ImageTestCase, test.NoDBTestCase):
         # a non-existent backend.
 
         base_dir = os.path.join(CONF.instances_path,
-                                CONF.image_cache_subdirectory_name)
+                                CONF.image_cache.subdirectory_name)
 
         # Lets assume the base image cache directory already exists
         existing = set([base_dir])

nova/tests/unit/virt/libvirt/test_imagecache.py

@@ -160,7 +160,7 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
                       lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
 
         found = os.path.join(CONF.instances_path,
-                             CONF.image_cache_subdirectory_name,
+                             CONF.image_cache.subdirectory_name,
                              'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
 
         image_cache_manager = imagecache.ImageCacheManager()
@@ -183,7 +183,7 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
                                  '10737418240'))
 
         found = os.path.join(CONF.instances_path,
-                             CONF.image_cache_subdirectory_name,
+                             CONF.image_cache.subdirectory_name,
                              'e97222e91fc4241f49a7f520d1dcf446751129b3_'
                              '10737418240')
 
@@ -205,7 +205,7 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
                       lambda x: 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
 
         found = os.path.join(CONF.instances_path,
-                             CONF.image_cache_subdirectory_name,
+                             CONF.image_cache.subdirectory_name,
                              'e97222e91fc4241f49a7f520d1dcf446751129b3_sm')
 
         image_cache_manager = imagecache.ImageCacheManager()
@@ -415,8 +415,8 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
         hashed_22 = '12c6fc06c99a462375eeb3f43dfd832b08ca9e17'
         hashed_42 = '92cfceb39d57d914ed8b14d0e37643de0797ae56'
 
-        self.flags(instances_path='/instance_path',
+        self.flags(instances_path='/instance_path')
-                   image_cache_subdirectory_name='_base')
+        self.flags(subdirectory_name='_base', group='image_cache')
 
         base_file_list = ['00000001',
                           'ephemeral_0_20_None',

nova/tests/unit/virt/test_imagecache.py

@@ -53,11 +53,12 @@ swap_bdm_256 = [block_device.BlockDeviceDict(
 class ImageCacheManagerTests(test.NoDBTestCase):
 
     def test_configurationi_defaults(self):
-        self.assertEqual(2400, CONF.image_cache_manager_interval)
+        self.assertEqual(2400, CONF.image_cache.manager_interval)
-        self.assertEqual('_base', CONF.image_cache_subdirectory_name)
+        self.assertEqual('_base', CONF.image_cache.subdirectory_name)
-        self.assertTrue(CONF.remove_unused_base_images)
+        self.assertTrue(CONF.image_cache.remove_unused_base_images)
-        self.assertEqual(24 * 3600,
+        self.assertEqual(
-                         CONF.remove_unused_original_minimum_age_seconds)
+            24 * 3600,
+            CONF.image_cache.remove_unused_original_minimum_age_seconds)
 
     def test_cache_manager(self):
         cache_manager = imagecache.ImageCacheManager()

nova/tests/unit/virt/vmwareapi/test_driver_api.py

@@ -213,8 +213,8 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
                    api_retry_count=1,
                    use_linked_clone=False, group='vmware')
         self.flags(enabled=False, group='vnc')
-        self.flags(image_cache_subdirectory_name='vmware_base',
+        self.flags(subdirectory_name='vmware_base', group='image_cache')
-                   my_ip='')
+        self.flags(my_ip='')
         self.user_id = 'fake'
         self.project_id = 'fake'
         self.context = context.RequestContext(self.user_id, self.project_id)
@@ -2021,7 +2021,7 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
 
     def test_image_aging_disabled(self):
         self._override_time()
-        self.flags(remove_unused_base_images=False)
+        self.flags(remove_unused_base_images=False, group='image_cache')
         self._create_vm()
         self._cached_files_exist()
         all_instances = []
@@ -2032,7 +2032,8 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
     def _image_aging_aged(self, aging_time=100):
         self._override_time()
         cur_time = datetime.datetime(2012, 11, 22, 12, 00, 10)
-        self.flags(remove_unused_original_minimum_age_seconds=aging_time)
+        self.flags(remove_unused_original_minimum_age_seconds=aging_time,
+                   group='image_cache')
         self._image_aging_image_marked_for_deletion()
         all_instances = []
         self.useFixture(utils_fixture.TimeFixture(cur_time))

nova/tests/unit/virt/vmwareapi/test_vmops.py

@@ -64,8 +64,8 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
         vmwareapi_fake.reset()
         stubs.set_stubs(self)
         self.flags(enabled=True, group='vnc')
-        self.flags(image_cache_subdirectory_name='vmware_base',
+        self.flags(subdirectory_name='vmware_base', group='image_cache')
-                   my_ip='',
+        self.flags(my_ip='',
                    flat_injected=True)
         self._context = context.RequestContext('fake_user', 'fake_project')
         self._session = driver.VMwareAPISession()
@@ -2723,19 +2723,19 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
         self.assertEqual('flavor-policy', extra_specs.storage_policy)
 
     def test_get_base_folder_not_set(self):
-        self.flags(image_cache_subdirectory_name='vmware_base')
+        self.flags(subdirectory_name='vmware_base', group='image_cache')
         base_folder = self._vmops._get_base_folder()
         self.assertEqual('vmware_base', base_folder)
 
     def test_get_base_folder_host_ip(self):
-        self.flags(my_ip='7.7.7.7',
+        self.flags(my_ip='7.7.7.7')
-                   image_cache_subdirectory_name='_base')
+        self.flags(subdirectory_name='_base', group='image_cache')
         base_folder = self._vmops._get_base_folder()
         self.assertEqual('7.7.7.7_base', base_folder)
 
     def test_get_base_folder_cache_prefix(self):
         self.flags(cache_prefix='my_prefix', group='vmware')
-        self.flags(image_cache_subdirectory_name='_base')
+        self.flags(subdirectory_name='_base', group='image_cache')
         base_folder = self._vmops._get_base_folder()
         self.assertEqual('my_prefix_base', base_folder)
 

nova/virt/hyperv/imagecache.py

@@ -171,7 +171,7 @@ class ImageCache(imagecache.ImageCacheManager):
                 # change the timestamp on the image so as to reflect the last
                 # time it was used
                 self._update_image_timestamp(img)
-            elif CONF.remove_unused_base_images:
+            elif CONF.image_cache.remove_unused_base_images:
                 self._remove_if_old_image(img)
 
     def _update_image_timestamp(self, image):
@@ -196,7 +196,8 @@ class ImageCache(imagecache.ImageCacheManager):
 
     def _remove_if_old_image(self, image):
         backing_files = self._get_image_backing_files(image)
-        max_age_seconds = CONF.remove_unused_original_minimum_age_seconds
+        max_age_seconds = (
+            CONF.image_cache.remove_unused_original_minimum_age_seconds)
 
         for img in backing_files:
             age_seconds = self._pathutils.get_age_of_file(img)

nova/virt/imagecache.py

@@ -29,7 +29,8 @@ class ImageCacheManager(object):
     """
 
     def __init__(self):
-        self.remove_unused_base_images = CONF.remove_unused_base_images
+        self.remove_unused_base_images = (
+            CONF.image_cache.remove_unused_base_images)
         self.resize_states = [task_states.RESIZE_PREP,
                               task_states.RESIZE_MIGRATING,
                               task_states.RESIZE_MIGRATED,

nova/virt/libvirt/driver.py

@@ -9658,7 +9658,7 @@ class LibvirtDriver(driver.ComputeDriver):
 
     def cache_image(self, context, image_id):
         cache_dir = os.path.join(CONF.instances_path,
-                                 CONF.image_cache_subdirectory_name)
+                                 CONF.image_cache.subdirectory_name)
         path = os.path.join(cache_dir,
                             imagecache.get_cache_fname(image_id))
         if os.path.exists(path):

nova/virt/libvirt/imagebackend.py

@@ -249,7 +249,7 @@ class Image(object):
         :size: Size of created image in bytes (optional)
         """
         base_dir = os.path.join(CONF.instances_path,
-                                CONF.image_cache_subdirectory_name)
+                                CONF.image_cache.subdirectory_name)
         if not os.path.exists(base_dir):
             fileutils.ensure_tree(base_dir)
         base = os.path.join(base_dir, filename)

nova/virt/libvirt/imagecache.py

@@ -137,7 +137,7 @@ class ImageCacheManager(imagecache.ImageCacheManager):
                     if backing_file:
                         backing_path = os.path.join(
                             CONF.instances_path,
-                            CONF.image_cache_subdirectory_name,
+                            CONF.image_cache.subdirectory_name,
                             backing_file)
                         if backing_path not in inuse_images:
                             inuse_images.append(backing_path)
@@ -236,15 +236,16 @@ class ImageCacheManager(imagecache.ImageCacheManager):
 
     def _remove_swap_file(self, base_file):
         """Remove a single swap base file if it is old enough."""
-        maxage = CONF.remove_unused_original_minimum_age_seconds
+        maxage = CONF.image_cache.remove_unused_original_minimum_age_seconds
 
         self._remove_old_enough_file(base_file, maxage, remove_lock=False)
 
     def _remove_base_file(self, base_file):
         """Remove a single base file if it is old enough."""
-        maxage = CONF.libvirt.remove_unused_resized_minimum_age_seconds
+        maxage = CONF.image_cache.remove_unused_resized_minimum_age_seconds
         if base_file in self.originals:
-            maxage = CONF.remove_unused_original_minimum_age_seconds
+            maxage = (
+                CONF.image_cache.remove_unused_original_minimum_age_seconds)
 
         self._remove_old_enough_file(base_file, maxage)
 
@@ -330,7 +331,7 @@ class ImageCacheManager(imagecache.ImageCacheManager):
         # created, but may remain from previous versions.
 
         base_dir = os.path.join(CONF.instances_path,
-                                CONF.image_cache_subdirectory_name)
+                                CONF.image_cache.subdirectory_name)
         if not os.path.exists(base_dir):
             LOG.debug('Skipping verification, no base directory at %s',
                       base_dir)

nova/virt/vmwareapi/imagecache.py

@@ -140,7 +140,8 @@ class ImageCacheManager(imagecache.ImageCacheManager):
     def _age_cached_images(self, context, datastore, dc_info,
                            ds_path):
         """Ages cached images."""
-        age_seconds = CONF.remove_unused_original_minimum_age_seconds
+        age_seconds = (
+            CONF.image_cache.remove_unused_original_minimum_age_seconds)
         unused_images = self.originals - self.used_images
         ds_browser = self._get_ds_browser(datastore.ref)
         for image in unused_images:

nova/virt/vmwareapi/vmops.py

@@ -152,14 +152,14 @@ class VMwareVMOps(object):
         # Enable more than one compute node to run on the same host
         if CONF.vmware.cache_prefix:
             base_folder = '%s%s' % (CONF.vmware.cache_prefix,
-                                    CONF.image_cache_subdirectory_name)
+                                    CONF.image_cache.subdirectory_name)
         # Ensure that the base folder is unique per compute node
-        elif CONF.remove_unused_base_images:
+        elif CONF.image_cache.remove_unused_base_images:
             base_folder = '%s%s' % (CONF.my_ip,
-                                    CONF.image_cache_subdirectory_name)
+                                    CONF.image_cache.subdirectory_name)
         else:
             # Aging disable ensures backward compatibility
-            base_folder = CONF.image_cache_subdirectory_name
+            base_folder = CONF.image_cache.subdirectory_name
         return base_folder
 
     def _extend_virtual_disk(self, instance, requested_size, name, dc_ref):
@@ -1855,7 +1855,7 @@ class VMwareVMOps(object):
         self._set_machine_id(client_factory, instance, network_info)
 
     def manage_image_cache(self, context, instances):
-        if not CONF.remove_unused_base_images:
+        if not CONF.image_cache.remove_unused_base_images:
             LOG.debug("Image aging disabled. Aging will not be done.")
             return
 

releasenotes/notes/image_cache-conf-opts-moved-e552e4a2d59e056e.yaml

@@ -0,0 +1,22 @@
+---
+deprecations:
+  - |
+    The following conf options have been moved to the ``[image_cache]`` group
+    and renamed accordingly. The old option paths are deprecated and will be
+    removed in a future release.
+
+    .. list-table::
+       :header-rows: 1
+
+       * - Deprecated Option
+         - New Option
+       * - ``[DEFAULT]image_cache_manager_interval``
+         - ``[image_cache]manager_interval``
+       * - ``[DEFAULT]image_cache_subdirectory_name``
+         - ``[image_cache]subdirectory_name``
+       * - ``[DEFAULT]remove_unused_base_images``
+         - ``[image_cache]remove_unused_base_images``
+       * - ``[DEFAULT]remove_unused_original_minimum_age_seconds``
+         - ``[image_cache]remove_unused_original_minimum_age_seconds``
+       * - ``[libvirt]remove_unused_resized_minimum_age_seconds``
+         - ``[image_cache]remove_unused_resized_minimum_age_seconds``