From 5cbe39aca96c7f2bfa3a49a9e964ae3f0123af81 Mon Sep 17 00:00:00 2001 From: Balazs Gibizer Date: Tue, 29 Apr 2025 15:43:47 +0200 Subject: [PATCH] Allow services to start with threading At the service startup nova need to initialize either the eventlet or the threading backend of oslo.service. So this patch reuses the existing logic behind OS_NOVA_DISABLE_EVENTLET_PATCHING. When OS_NOVA_DISABLE_EVENTLET_PATCHING env variable is set to true the service will select the threading backend otherwise the eventlet backend. Also to avoid later monkey patch calls to invalidated the selection if the threading backend is selected then the monkey_patch code is poisoned. This patch also makes sure that oslo.messaging also initialized with the matching executor backend. As this is the last step to make nova-scheduler run in threading mode this patch adds a release notes as well. Change-Id: I6e2e6a43df78d23580b5e7402352a5036100ab36 Signed-off-by: Balazs Gibizer --- doc/source/conf.py | 8 ---- nova/monkey_patch.py | 48 +++++++++++++++---- nova/rpc.py | 4 +- nova/tests/unit/test_rpc.py | 23 +++++++++ nova/tests/unit/test_utils.py | 40 ++++++++++++++++ ...eaded-nova-scheduler-dd4649b987f33025.yaml | 9 ++++ requirements.txt | 2 +- 7 files changed, 116 insertions(+), 18 deletions(-) create mode 100644 releasenotes/notes/threaded-nova-scheduler-dd4649b987f33025.yaml diff --git a/doc/source/conf.py b/doc/source/conf.py index ad30372563..cd84b1f99e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -188,11 +188,3 @@ openstackdocs_projects = [ 'watcher', ] # -- Custom extensions -------------------------------------------------------- - -# NOTE(mdbooth): (2019-03-20) Sphinx loads policies defined in setup.cfg, which -# includes the placement policy at nova/api/openstack/placement/policies.py. -# Loading this imports nova/api/openstack/__init__.py, which imports -# nova.monkey_patch, which will do eventlet monkey patching to the sphinx -# process. As well as being unnecessary and a bad idea, this breaks on -# python3.6 (but not python3.7), so don't do that. -os.environ['OS_NOVA_DISABLE_EVENTLET_PATCHING'] = '1' diff --git a/nova/monkey_patch.py b/nova/monkey_patch.py index c196ac8ed1..c3bc62dea9 100644 --- a/nova/monkey_patch.py +++ b/nova/monkey_patch.py @@ -29,7 +29,8 @@ def is_patched(): def _monkey_patch(): if is_patched(): - return + return False + # NOTE(mdbooth): Anything imported here will not be monkey patched. It is # important to take care not to import anything here which requires monkey # patching. @@ -68,14 +69,45 @@ def _monkey_patch(): "importing and not executing nova code.", ', '.join(problems)) + return True + def patch(): - # NOTE(mdbooth): This workaround is required to avoid breaking sphinx. See - # separate comment in doc/source/conf.py. It may also be useful for other - # non-nova utilities. Ideally the requirement for this workaround will be - # removed as soon as possible, so do not rely on, or extend it. if (os.environ.get('OS_NOVA_DISABLE_EVENTLET_PATCHING', '').lower() not in ('1', 'true', 'yes')): - _monkey_patch() - global MONKEY_PATCHED - MONKEY_PATCHED = True + + if _monkey_patch(): + global MONKEY_PATCHED + MONKEY_PATCHED = True + + import oslo_service.backend as service + service.init_backend(service.BackendType.EVENTLET) + from oslo_log import log as logging + LOG = logging.getLogger(__name__) + LOG.info("Service is starting with Eventlet based service backend") + else: + # We asked not to monkey patch so we will run in native threading mode + import oslo_service.backend as service + # NOTE(gibi): This will raise if the backend is already initialized + # with Eventlet + service.init_backend(service.BackendType.THREADING) + + # NOTE(gibi): We were asked not to monkey patch. Let's enforce it by + # removing the possibility to monkey_patch accidentally + def poison(*args, **kwargs): + raise RuntimeError( + "The service is started with native threading via " + "OS_NOVA_DISABLE_EVENTLET_PATCHING set to '%s', but then the " + "service tried to call eventlet.monkey_patch(). This is a " + "bug." + % os.environ.get('OS_NOVA_DISABLE_EVENTLET_PATCHING', '')) + + import eventlet + eventlet.monkey_patch = poison + eventlet.patcher.monkey_patch = poison + + from oslo_log import log as logging + LOG = logging.getLogger(__name__) + LOG.warning( + "Service is starting with native threading. This is currently " + "experimental. Do not use it in production.") diff --git a/nova/rpc.py b/nova/rpc.py index 7a92650414..b2a3997982 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -25,6 +25,7 @@ import nova.conf import nova.context import nova.exception from nova.i18n import _ +from nova import utils __all__ = [ 'init', @@ -217,10 +218,11 @@ def get_server(target, endpoints, serializer=None): else: serializer = RequestContextSerializer(serializer) access_policy = dispatcher.DefaultRPCAccessPolicy + exc = "threading" if utils.concurrency_mode_threading() else "eventlet" return messaging.get_rpc_server(TRANSPORT, target, endpoints, - executor='eventlet', + executor=exc, serializer=serializer, access_policy=access_policy) diff --git a/nova/tests/unit/test_rpc.py b/nova/tests/unit/test_rpc.py index 40a914b5f7..6eccdd04fd 100644 --- a/nova/tests/unit/test_rpc.py +++ b/nova/tests/unit/test_rpc.py @@ -250,6 +250,29 @@ class TestRPC(test.NoDBTestCase): access_policy=access_policy) self.assertEqual('server', server) + @mock.patch( + 'nova.utils.concurrency_mode_threading', + new=mock.Mock(return_value=True)) + @mock.patch.object(rpc, 'TRANSPORT') + @mock.patch.object(rpc, 'profiler', None) + @mock.patch.object(rpc, 'RequestContextSerializer') + @mock.patch.object(messaging, 'get_rpc_server') + def test_get_server_threading(self, mock_get, mock_ser, mock_TRANSPORT): + ser = mock.Mock() + tgt = mock.Mock() + ends = mock.Mock() + mock_ser.return_value = ser + mock_get.return_value = 'server' + + server = rpc.get_server(tgt, ends, serializer='foo') + + mock_ser.assert_called_once_with('foo') + access_policy = dispatcher.DefaultRPCAccessPolicy + mock_get.assert_called_once_with(mock_TRANSPORT, tgt, ends, + executor='threading', serializer=ser, + access_policy=access_policy) + self.assertEqual('server', server) + @mock.patch.object(rpc, 'TRANSPORT') @mock.patch.object(rpc, 'profiler', mock.Mock()) @mock.patch.object(rpc, 'ProfilerRequestContextSerializer') diff --git a/nova/tests/unit/test_utils.py b/nova/tests/unit/test_utils.py index 51fc093149..63b0dbad97 100644 --- a/nova/tests/unit/test_utils.py +++ b/nova/tests/unit/test_utils.py @@ -14,6 +14,7 @@ import datetime import hashlib +import os import threading from unittest import mock @@ -26,11 +27,13 @@ from openstack import exceptions as sdk_exc from oslo_config import cfg from oslo_context import context as common_context from oslo_context import fixture as context_fixture +import oslo_service.backend as oslo_backend from oslo_utils import encodeutils from oslo_utils import fixture as utils_fixture from nova import context from nova import exception +from nova import monkey_patch from nova.objects import base as obj_base from nova.objects import instance as instance_obj from nova.objects import service as service_obj @@ -1651,3 +1654,40 @@ class ExecutorStatsTestCase(test.NoDBTestCase): utils.spawn(self._task_finishes).result() mock_info.assert_not_called() + + +class OsloServiceBackendSelectionTestCase(test.NoDBTestCase): + def setUp(self): + # NOTE(gibi): We need this as the base test class would trigger + # monkey patching and would prevent us to test the threading code path + self.useFixture( + fixtures.MonkeyPatch( + "nova.monkey_patch._monkey_patch", lambda: True)) + super().setUp() + origi = monkey_patch.MONKEY_PATCHED + monkey_patch.MONKEY_PATCHED = False + + def reset(): + monkey_patch.MONKEY_PATCHED = origi + self.addCleanup(reset) + + @mock.patch('oslo_service.backend.init_backend') + def test_eventlet_selected(self, init_backend): + monkey_patch.patch() + + init_backend.assert_called_once_with(oslo_backend.BackendType.EVENTLET) + + @mock.patch('oslo_service.backend.init_backend') + @mock.patch.dict(os.environ, {"OS_NOVA_DISABLE_EVENTLET_PATCHING": "true"}) + def test_threading_selected_monkey_patching_poisoned(self, init_backend): + monkey_patch.patch() + + init_backend.assert_called_once_with( + oslo_backend.BackendType.THREADING) + import eventlet + ex = self.assertRaises(RuntimeError, eventlet.monkey_patch) + self.assertEqual( + "The service is started with native threading via " + "OS_NOVA_DISABLE_EVENTLET_PATCHING set to 'true', but then the " + "service tried to call eventlet.monkey_patch(). This is a bug.", + str(ex)) diff --git a/releasenotes/notes/threaded-nova-scheduler-dd4649b987f33025.yaml b/releasenotes/notes/threaded-nova-scheduler-dd4649b987f33025.yaml new file mode 100644 index 0000000000..2923eb9e2f --- /dev/null +++ b/releasenotes/notes/threaded-nova-scheduler-dd4649b987f33025.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + The nova-scheduler now can be run in native threading mode instead + of with eventlet. This is an experimental feature that is disabled by + default. Please read the + `concurrency `__ + guide for more details. + diff --git a/requirements.txt b/requirements.txt index 11b73cd0e5..89873efe09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ oslo.messaging>=14.1.0 # Apache-2.0 oslo.policy>=4.5.0 # Apache-2.0 oslo.privsep>=2.6.2 # Apache-2.0 oslo.i18n>=5.1.0 # Apache-2.0 -oslo.service>=2.8.0 # Apache-2.0 +oslo.service[threading]>=4.2.0 # Apache-2.0 rfc3986>=1.2.0 # Apache-2.0 oslo.middleware>=3.31.0 # Apache-2.0 psutil>=3.2.2 # BSD