diff --git a/doc/source/cli/nova-manage.rst b/doc/source/cli/nova-manage.rst index 8354e9bc2b..4dc614db2a 100644 --- a/doc/source/cli/nova-manage.rst +++ b/doc/source/cli/nova-manage.rst @@ -227,6 +227,7 @@ db archive_deleted_rows nova-manage db archive_deleted_rows [--max_rows ] [--verbose] [--until-complete] [--before ] [--purge] [--all-cells] [--task-log] + [--sleep] Move deleted rows from production tables to shadow tables. Note that the corresponding rows in the ``instance_mappings``, ``request_specs`` and @@ -240,7 +241,7 @@ stopping at 0, or use the :option:`--until-complete` option. .. versionchanged:: 24.0.0 (Xena) - Added :option:`--task-log` option. + Added :option:`--task-log`, :option:`--sleep` options. .. rubric:: Options @@ -295,7 +296,12 @@ stopping at 0, or use the :option:`--until-complete` option. record data via the `/os-instance_usage_audit_log`__ API (example: Telemetry). -.. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log + .. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log + +.. option:: --sleep + + The amount of time in seconds to sleep between batches when + :option:`--until-complete` is used. Defaults to 0. .. rubric:: Return codes diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 221fed4437..95c455d948 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -26,6 +26,7 @@ import functools import os import re import sys +import time import traceback from urllib import parse as urlparse @@ -249,9 +250,14 @@ class DbCommands(object): '``--before`` option to avoid races for those consuming ' '``task_log`` record data via the ' '``/os-instance_usage_audit_log`` API (example: Telemetry).')) - def archive_deleted_rows(self, max_rows=1000, verbose=False, - until_complete=False, purge=False, - before=None, all_cells=False, task_log=False): + @args('--sleep', type=int, metavar='', dest='sleep', + help='The amount of time in seconds to sleep between batches when ' + '``--until-complete`` is used. Defaults to 0.') + def archive_deleted_rows( + self, max_rows=1000, verbose=False, + until_complete=False, purge=False, + before=None, all_cells=False, task_log=False, sleep=0, + ): """Move deleted rows from production tables to shadow tables. Returns 0 if nothing was archived, 1 if some number of rows were @@ -344,7 +350,8 @@ class DbCommands(object): verbose, before_date, cell_name, - task_log) + task_log, + sleep) except KeyboardInterrupt: interrupt = True break @@ -377,8 +384,10 @@ class DbCommands(object): # NOTE(danms): Return nonzero if we archived something return int(bool(table_to_rows_archived)) - def _do_archive(self, table_to_rows_archived, cctxt, max_rows, - until_complete, verbose, before_date, cell_name, task_log): + def _do_archive( + self, table_to_rows_archived, cctxt, max_rows, + until_complete, verbose, before_date, cell_name, task_log, sleep, + ): """Helper function for archiving deleted rows for a cell. This will archive deleted rows for a cell database and remove the @@ -398,6 +407,8 @@ class DbCommands(object): :param cell_name: Name of the cell or None if not archiving across all cells :param task_log: Whether to archive task_log table rows + :param sleep: The amount of time in seconds to sleep between batches + when ``until_complete`` is True. """ ctxt = context.get_admin_context() while True: @@ -437,6 +448,8 @@ class DbCommands(object): break if verbose: sys.stdout.write('.') + # Optionally sleep between batches to throttle the archiving. + time.sleep(sleep) return total_rows_archived @args('--before', metavar='', dest='before', diff --git a/nova/tests/unit/cmd/test_manage.py b/nova/tests/unit/cmd/test_manage.py index fdf0a6d3aa..4f9c732c94 100644 --- a/nova/tests/unit/cmd/test_manage.py +++ b/nova/tests/unit/cmd/test_manage.py @@ -315,17 +315,20 @@ Archiving.....complete # Tests that we get table output. self._test_archive_deleted_rows(verbose=True) + @mock.patch('time.sleep') @mock.patch.object(db, 'archive_deleted_rows') @mock.patch.object(objects.CellMappingList, 'get_all') def test_archive_deleted_rows_until_complete(self, mock_get_all, - mock_db_archive, - verbose=False): + mock_db_archive, mock_sleep, + verbose=False, + sleep=0): mock_db_archive.side_effect = [ ({'instances': 10, 'instance_extra': 5}, list(), 15), ({'instances': 5, 'instance_faults': 1}, list(), 6), ({}, list(), 0)] result = self.commands.archive_deleted_rows(20, verbose=verbose, - until_complete=True) + until_complete=True, + sleep=sleep) self.assertEqual(1, result) if verbose: expected = """\ @@ -353,10 +356,15 @@ Archiving.....complete test.MatchType(context.RequestContext), 20, before=None, task_log=False), ]) + self.assertEqual(2, mock_sleep.call_count) + mock_sleep.assert_has_calls([mock.call(sleep), mock.call(sleep)]) def test_archive_deleted_rows_until_complete_quiet(self): self.test_archive_deleted_rows_until_complete(verbose=False) + def test_archive_deleted_rows_until_complete_sleep(self): + self.test_archive_deleted_rows_until_complete(sleep=30) + @mock.patch('nova.db.main.api.purge_shadow_tables') @mock.patch.object(db, 'archive_deleted_rows') @mock.patch.object(objects.CellMappingList, 'get_all') diff --git a/releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml b/releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml new file mode 100644 index 0000000000..9715567fd0 --- /dev/null +++ b/releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + A ``--sleep`` option has been added to the ``nova-manage db + archive_deleted_rows`` CLI. When this command is run with the + ``--until-complete`` option, the process will archive rows in batches + in a tight loop, which can cause problems in busy environments where + the aggressive archiving interferes with other requests trying to write + to the database. The ``--sleep`` option can be used to specify a time to + sleep between batches of rows while archiving with ``--until-complete``, + allowing the process to be throttled.