From 9f64b9900e705e040aadb07ec3b9db140b4fc11b Mon Sep 17 00:00:00 2001 From: Dustin Cowles Date: Thu, 11 Apr 2019 15:12:19 -0700 Subject: [PATCH] Introduces the openstacksdk to nova Enables the use of the sdk instead of ksa adapter or python-*client. It is provided by a get_sdk_adapter method which constructs an authenticated SDK Connection object using provided service configuration. This change should be transparent to operators of services which already use ksa as get_sdk_adapter uses the same conf options from keystoneauth1. Blueprint: openstacksdk-in-nova Co-Authored-By: Dustin Cowles Change-Id: I49f364e01e2a18de0c95674654fc72acea019e76 --- lower-constraints.txt | 4 +- nova/test.py | 2 + nova/tests/fixtures.py | 25 +++++ nova/tests/unit/test_utils.py | 193 ++++++++++++++++++++++++++++++---- nova/utils.py | 78 ++++++++++---- requirements.txt | 3 +- 6 files changed, 259 insertions(+), 46 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 8a1812c568..7c682e63a8 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -63,11 +63,11 @@ netaddr==0.7.18 netifaces==0.10.4 networkx==1.11 numpy==1.14.2 -openstacksdk==0.12.0 +openstacksdk==0.31.0 os-brick==2.6.1 os-client-config==1.29.0 os-resource-classes==0.1.0 -os-service-types==1.2.0 +os-service-types==1.7.0 os-traits==0.15.0 os-vif==1.14.0 os-win==3.0.0 diff --git a/nova/test.py b/nova/test.py index 04062d1f58..812ff435dc 100644 --- a/nova/test.py +++ b/nova/test.py @@ -206,6 +206,8 @@ class TestCase(testtools.TestCase): os.environ.get('OS_TEST_TIMEOUT', 0), self.TIMEOUT_SCALING_FACTOR)) + self.useFixture(nova_fixtures.OpenStackSDKFixture()) + self.useFixture(fixtures.NestedTempfile()) self.useFixture(fixtures.TempHomeDir()) self.useFixture(log_fixture.get_logging_handle_error_fixture()) diff --git a/nova/tests/fixtures.py b/nova/tests/fixtures.py index 62211b98eb..92a42a1fc9 100644 --- a/nova/tests/fixtures.py +++ b/nova/tests/fixtures.py @@ -2104,3 +2104,28 @@ class AvailabilityZoneFixture(fixtures.Fixture): self.useFixture(fixtures.MonkeyPatch( 'nova.availability_zones.get_instance_availability_zone', fake_get_instance_availability_zone)) + + +class KSAFixture(fixtures.Fixture): + """Lets us initialize an openstack.connection.Connection by stubbing the + auth plugin. + """ + def setUp(self): + super(KSAFixture, self).setUp() + self.mock_load_auth = self.useFixture(fixtures.MockPatch( + 'keystoneauth1.loading.load_auth_from_conf_options')).mock + self.mock_load_sess = self.useFixture(fixtures.MockPatch( + 'keystoneauth1.loading.load_session_from_conf_options')).mock + # For convenience, an attribute for the "Session" itself + self.mock_session = self.mock_load_sess.return_value + + +class OpenStackSDKFixture(fixtures.Fixture): + # This satisfies tests that happen to run through get_sdk_adapter but don't + # care about the adapter itself (default mocks are fine). + # TODO(efried): Get rid of this and use fixtures from openstacksdk once + # https://storyboard.openstack.org/#!/story/2005475 is resolved. + def setUp(self): + super(OpenStackSDKFixture, self).setUp() + self.useFixture(fixtures.MockPatch( + 'keystoneauth1.adapter.Adapter.get_api_major_version')) diff --git a/nova/tests/unit/test_utils.py b/nova/tests/unit/test_utils.py index 81c30a1514..af1fc91105 100644 --- a/nova/tests/unit/test_utils.py +++ b/nova/tests/unit/test_utils.py @@ -40,6 +40,7 @@ from nova import exception from nova.objects import base as obj_base from nova.objects import instance as instance_obj from nova import test +from nova.tests import fixtures as nova_fixtures from nova.tests.unit.objects import test_objects from nova.tests.unit import utils as test_utils from nova import utils @@ -1207,28 +1208,24 @@ class GetKSAAdapterTestCase(test.NoDBTestCase): self.auth = mock.create_autospec(ks_identity.BaseIdentityPlugin, instance=True) - load_sess_p = mock.patch( - 'keystoneauth1.loading.load_session_from_conf_options') - self.addCleanup(load_sess_p.stop) - self.load_sess = load_sess_p.start() - self.load_sess.return_value = self.sess - load_adap_p = mock.patch( 'keystoneauth1.loading.load_adapter_from_conf_options') self.addCleanup(load_adap_p.stop) self.load_adap = load_adap_p.start() - load_auth_p = mock.patch( - 'keystoneauth1.loading.load_auth_from_conf_options') - self.addCleanup(load_auth_p.stop) - self.load_auth = load_auth_p.start() - self.load_auth.return_value = self.auth + ksa_fixture = self.useFixture(nova_fixtures.KSAFixture()) + self.mock_ksa_load_auth = ksa_fixture.mock_load_auth + self.mock_ksa_load_sess = ksa_fixture.mock_load_sess + self.mock_ksa_session = ksa_fixture.mock_session + + self.mock_ksa_load_auth.return_value = self.auth + self.mock_ksa_load_sess.return_value = self.sess def test_bogus_service_type(self): self.assertRaises(exception.ConfGroupForServiceTypeNotFound, utils.get_ksa_adapter, 'bogus') - self.load_auth.assert_not_called() - self.load_sess.assert_not_called() + self.mock_ksa_load_auth.assert_not_called() + self.mock_ksa_load_sess.assert_not_called() self.load_adap.assert_not_called() def test_all_params(self): @@ -1238,9 +1235,9 @@ class GetKSAAdapterTestCase(test.NoDBTestCase): # Returned the result of load_adapter_from_conf_options self.assertEqual(self.load_adap.return_value, ret) # Because we supplied ksa_auth, load_auth* not called - self.load_auth.assert_not_called() + self.mock_ksa_load_auth.assert_not_called() # Ditto ksa_session/load_session* - self.load_sess.assert_not_called() + self.mock_ksa_load_sess.assert_not_called() # load_adapter* called with what we passed in (and the right group) self.load_adap.assert_called_once_with( utils.CONF, 'glance', session='sess', auth='auth', @@ -1252,9 +1249,9 @@ class GetKSAAdapterTestCase(test.NoDBTestCase): # Returned the result of load_adapter_from_conf_options self.assertEqual(self.load_adap.return_value, ret) # Because ksa_auth found in ksa_session, load_auth* not called - self.load_auth.assert_not_called() + self.mock_ksa_load_auth.assert_not_called() # Because we supplied ksa_session, load_session* not called - self.load_sess.assert_not_called() + self.mock_ksa_load_sess.assert_not_called() # load_adapter* called with the auth from the session self.load_adap.assert_called_once_with( utils.CONF, 'ironic', session=self.sess, auth='auth', @@ -1265,10 +1262,10 @@ class GetKSAAdapterTestCase(test.NoDBTestCase): # Returned the result of load_adapter_from_conf_options self.assertEqual(self.load_adap.return_value, ret) # Had to load the auth - self.load_auth.assert_called_once_with(utils.CONF, 'cinder') + self.mock_ksa_load_auth.assert_called_once_with(utils.CONF, 'cinder') # Had to load the session, passed in the loaded auth - self.load_sess.assert_called_once_with(utils.CONF, 'cinder', - auth=self.auth) + self.mock_ksa_load_sess.assert_called_once_with(utils.CONF, 'cinder', + auth=self.auth) # load_adapter* called with the loaded auth & session self.load_adap.assert_called_once_with( utils.CONF, 'cinder', session=self.sess, auth=self.auth, @@ -1443,3 +1440,159 @@ class TestResourceClassNormalize(test.NoDBTestCase): """ name = u'Fu\xdfball' self.assertEqual(u'CUSTOM_FU_BALL', utils.normalize_rc_name(name)) + + +class TestGetConfGroup(test.NoDBTestCase): + """Tests for nova.utils._get_conf_group""" + @mock.patch('nova.utils.CONF') + @mock.patch('nova.utils._SERVICE_TYPES.get_project_name') + def test__get_conf_group(self, mock_get_project_name, mock_conf): + test_conf_grp = 'test_confgrp' + test_service_type = 'test_service_type' + mock_get_project_name.return_value = test_conf_grp + + # happy path + mock_conf.test_confgrp = None + actual_conf_grp = utils._get_conf_group(test_service_type) + self.assertEqual(test_conf_grp, actual_conf_grp) + mock_get_project_name.assert_called_once_with(test_service_type) + + # service type as the conf group + del mock_conf.test_confgrp + mock_conf.test_service_type = None + actual_conf_grp = utils._get_conf_group(test_service_type) + self.assertEqual(test_service_type, actual_conf_grp) + + @mock.patch('nova.utils._SERVICE_TYPES.get_project_name') + def test__get_conf_group_fail(self, mock_get_project_name): + test_service_type = 'test_service_type' + + # not confgrp + mock_get_project_name.return_value = None + self.assertRaises(exception.ConfGroupForServiceTypeNotFound, + utils._get_conf_group, None) + + # not hasattr + mock_get_project_name.return_value = 'test_fail' + self.assertRaises(exception.ConfGroupForServiceTypeNotFound, + utils._get_conf_group, test_service_type) + + +class TestGetAuthAndSession(test.NoDBTestCase): + """Tests for nova.utils._get_auth_and_session""" + def setUp(self): + super(TestGetAuthAndSession, self).setUp() + + self.test_auth = 'test_auth' + self.test_session = 'test_session' + self.test_session_auth = 'test_session_auth' + self.test_confgrp = 'test_confgrp' + self.mock_session = mock.Mock() + self.mock_session.auth = self.test_session_auth + + @mock.patch('nova.utils.ks_loading.load_auth_from_conf_options') + @mock.patch('nova.utils.ks_loading.load_session_from_conf_options') + def test_auth_and_session(self, mock_load_session, mock_load_auth): + # yes auth, yes session + actual = utils._get_auth_and_session(self.test_confgrp, + ksa_auth=self.test_auth, + ksa_session=self.test_session) + self.assertEqual(actual, (self.test_auth, self.test_session)) + mock_load_session.assert_not_called() + mock_load_auth.assert_not_called() + + @mock.patch('nova.utils.ks_loading.load_auth_from_conf_options') + @mock.patch('nova.utils.ks_loading.load_session_from_conf_options') + @mock.patch('nova.utils.CONF') + def test_no_session(self, mock_CONF, mock_load_session, mock_load_auth): + # yes auth, no session + mock_load_session.return_value = self.test_session + + actual = utils._get_auth_and_session(self.test_confgrp, + ksa_auth=self.test_auth, + ksa_session=None) + + self.assertEqual(actual, (self.test_auth, self.test_session)) + mock_load_session.assert_called_once_with(mock_CONF, self.test_confgrp, + auth=self.test_auth) + mock_load_auth.assert_not_called() + + @mock.patch('nova.utils.ks_loading.load_auth_from_conf_options') + @mock.patch('nova.utils.ks_loading.load_session_from_conf_options') + def test_no_auth(self, mock_load_session, mock_load_auth): + # no auth, yes session, yes session.auth + actual = utils._get_auth_and_session(self.test_confgrp, ksa_auth=None, + ksa_session=self.mock_session) + self.assertEqual(actual, (self.test_session_auth, self.mock_session)) + mock_load_session.assert_not_called() + mock_load_auth.assert_not_called() + + @mock.patch('nova.utils.ks_loading.load_auth_from_conf_options') + @mock.patch('nova.utils.ks_loading.load_session_from_conf_options') + @mock.patch('nova.utils.CONF') + def test_no_auth_no_sauth(self, mock_CONF, mock_load_session, + mock_load_auth): + # no auth, yes session, no session.auth + mock_load_auth.return_value = self.test_auth + self.mock_session.auth = None + actual = utils._get_auth_and_session(self.test_confgrp, ksa_auth=None, + ksa_session=self.mock_session) + self.assertEqual(actual, (self.test_auth, self.mock_session)) + mock_load_session.assert_not_called() + mock_load_auth.assert_called_once_with(mock_CONF, self.test_confgrp) + + @mock.patch('nova.utils.ks_loading.load_auth_from_conf_options') + @mock.patch('nova.utils.ks_loading.load_session_from_conf_options') + @mock.patch('nova.utils.CONF') + def test__get_auth_and_session(self, mock_CONF, mock_load_session, + mock_load_auth): + # no auth, no session + mock_load_auth.return_value = self.test_auth + mock_load_session.return_value = self.test_session + actual = utils._get_auth_and_session(self.test_confgrp, ksa_auth=None, + ksa_session=None) + self.assertEqual(actual, (self.test_auth, self.test_session)) + mock_load_session.assert_called_once_with(mock_CONF, self.test_confgrp, + auth=self.test_auth) + mock_load_auth.assert_called_once_with(mock_CONF, self.test_confgrp) + + +class TestGetSDKAdapter(test.NoDBTestCase): + """Tests for nova.utils.get_sdk_adapter""" + @mock.patch('nova.utils._get_conf_group') + @mock.patch('nova.utils._get_auth_and_session') + @mock.patch('nova.utils.connection.Connection') + @mock.patch('nova.utils.CONF') + def test_get_sdk_adapter(self, mock_conf, mock_connection, + mock_get_auth_sess, mock_get_confgrp): + service_type = 'test_service' + mock_conn = mock.Mock() + mock_proxy = mock.Mock() + setattr(mock_conn, service_type, mock_proxy) + mock_connection.return_value = mock_conn + mock_session = mock.Mock() + mock_get_auth_sess.return_value = (None, mock_session) + mock_get_confgrp.return_value = mock_confgrp = mock.Mock() + + actual = utils.get_sdk_adapter(service_type) + + self.assertEqual(actual, mock_proxy) + mock_get_confgrp.assert_called_once_with(service_type) + mock_get_auth_sess.assert_called_once_with(mock_confgrp) + mock_connection.assert_called_once_with(session=mock_session, + oslo_conf=mock_conf) + + @mock.patch('nova.utils._get_conf_group') + @mock.patch('nova.utils._get_auth_and_session') + @mock.patch('nova.utils.connection.Connection') + def test_get_sdk_adapter_fail(self, mock_connection, mock_get_auth_sess, + mock_get_confgrp): + service_type = 'test_service' + mock_get_confgrp.side_effect = \ + exception.ConfGroupForServiceTypeNotFound(stype=service_type) + + self.assertRaises(exception.ConfGroupForServiceTypeNotFound, + utils.get_sdk_adapter, service_type) + mock_get_confgrp.assert_called_once_with(service_type) + mock_connection.assert_not_called() + mock_get_auth_sess.assert_not_called() diff --git a/nova/utils.py b/nova/utils.py index 71d50f2d4b..05658f2f3e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -34,6 +34,7 @@ import eventlet from keystoneauth1 import exceptions as ks_exc from keystoneauth1 import loading as ks_loading import netaddr +from openstack import connection import os_resource_classes as orc from os_service_types import service_types from oslo_concurrency import lockutils @@ -1158,6 +1159,38 @@ def strtime(at): return at.strftime("%Y-%m-%dT%H:%M:%S.%f") +def _get_conf_group(service_type): + # Get the conf group corresponding to the service type. + confgrp = _SERVICE_TYPES.get_project_name(service_type) + if not confgrp or not hasattr(CONF, confgrp): + # Try the service type as the conf group. This is necessary for e.g. + # placement, while it's still part of the nova project. + # Note that this might become the first thing we try if/as we move to + # using service types for conf group names in general. + confgrp = service_type + if not confgrp or not hasattr(CONF, confgrp): + raise exception.ConfGroupForServiceTypeNotFound(stype=service_type) + return confgrp + + +def _get_auth_and_session(confgrp, ksa_auth=None, ksa_session=None): + # Ensure we have an auth. + # NOTE(efried): This could be None, and that could be okay - e.g. if the + # result is being used for get_endpoint() and the conf only contains + # endpoint_override. + if not ksa_auth: + if ksa_session and ksa_session.auth: + ksa_auth = ksa_session.auth + else: + ksa_auth = ks_loading.load_auth_from_conf_options(CONF, confgrp) + + if not ksa_session: + ksa_session = ks_loading.load_session_from_conf_options( + CONF, confgrp, auth=ksa_auth) + + return ksa_auth, ksa_session + + def get_ksa_adapter(service_type, ksa_auth=None, ksa_session=None, min_version=None, max_version=None): """Construct a keystoneauth1 Adapter for a given service type. @@ -1191,36 +1224,35 @@ def get_ksa_adapter(service_type, ksa_auth=None, ksa_session=None, :raise: ConfGroupForServiceTypeNotFound If no conf group name could be found for the specified service_type. """ - # Get the conf group corresponding to the service type. - confgrp = _SERVICE_TYPES.get_project_name(service_type) - if not confgrp or not hasattr(CONF, confgrp): - # Try the service type as the conf group. This is necessary for e.g. - # placement, while it's still part of the nova project. - # Note that this might become the first thing we try if/as we move to - # using service types for conf group names in general. - confgrp = service_type - if not confgrp or not hasattr(CONF, confgrp): - raise exception.ConfGroupForServiceTypeNotFound(stype=service_type) + confgrp = _get_conf_group(service_type) - # Ensure we have an auth. - # NOTE(efried): This could be None, and that could be okay - e.g. if the - # result is being used for get_endpoint() and the conf only contains - # endpoint_override. - if not ksa_auth: - if ksa_session and ksa_session.auth: - ksa_auth = ksa_session.auth - else: - ksa_auth = ks_loading.load_auth_from_conf_options(CONF, confgrp) - - if not ksa_session: - ksa_session = ks_loading.load_session_from_conf_options( - CONF, confgrp, auth=ksa_auth) + ksa_auth, ksa_session = _get_auth_and_session( + confgrp, ksa_auth, ksa_session) return ks_loading.load_adapter_from_conf_options( CONF, confgrp, session=ksa_session, auth=ksa_auth, min_version=min_version, max_version=max_version, raise_exc=False) +def get_sdk_adapter(service_type): + """Construct an openstacksdk-brokered Adapter for a given service type. + + We expect to find a conf group whose name corresponds to the service_type's + project according to the service-types-authority. That conf group must + provide ksa auth, session, and adapter options. + + :param service_type: String name of the service type for which the Adapter + is to be constructed. + :return: An openstack.proxy.Proxy object for the specified service_type. + :raise: ConfGroupForServiceTypeNotFound If no conf group name could be + found for the specified service_type. + """ + confgrp = _get_conf_group(service_type) + _, sess = _get_auth_and_session(confgrp) + conn = connection.Connection(session=sess, oslo_conf=CONF) + return getattr(conn, service_type) + + def get_endpoint(ksa_adapter): """Get the endpoint URL represented by a keystoneauth1 Adapter. diff --git a/requirements.txt b/requirements.txt index 9d3d537410..703f168e5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -66,8 +66,9 @@ tooz>=1.58.0 # Apache-2.0 cursive>=0.2.1 # Apache-2.0 pypowervm>=1.1.15 # Apache-2.0 retrying>=1.3.3,!=1.3.0 # Apache-2.0 -os-service-types>=1.2.0 # Apache-2.0 +os-service-types>=1.7.0 # Apache-2.0 taskflow>=2.16.0 # Apache-2.0 python-dateutil>=2.5.3 # BSD zVMCloudConnector>=1.3.0;sys_platform!='win32' # Apache 2.0 License futurist>=1.8.0 # Apache-2.0 +openstacksdk>=0.31.0 # Apache-2.0