From c9a64996ecc317b2c05d688e0f5d31c37122ca01 Mon Sep 17 00:00:00 2001 From: Dao Cong Tien Date: Fri, 10 Jun 2016 15:02:43 +0700 Subject: [PATCH] Ironic Driver: override get_serial_console() This implements Nova-compatible serial console for Ironic in Nova scope. [Nova] blueprint ironic-serial-console-support [Ironic] spec https://review.openstack.org/319505 [Ironic] related patches https://review.openstack.org/#/c/293873/ https://review.openstack.org/#/c/328168/ [Ironic] Related-Bug: #1553083 Change-Id: I38e803021d71fc0760a8ae99b3e97dd0aecb5088 --- doc/source/support-matrix.ini | 2 +- nova/conf/ironic.py | 6 + nova/conf/serial_console.py | 4 +- nova/tests/unit/virt/ironic/test_driver.py | 310 ++++++++++++++++++ nova/virt/ironic/driver.py | 156 +++++++++ ...rial-console-support-82632bd4db6d1fda.yaml | 9 + 6 files changed, 484 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/ironic-serial-console-support-82632bd4db6d1fda.yaml diff --git a/doc/source/support-matrix.ini b/doc/source/support-matrix.ini index 4dc02cfa10..340ca95235 100644 --- a/doc/source/support-matrix.ini +++ b/doc/source/support-matrix.ini @@ -810,7 +810,7 @@ driver-impl-libvirt-lxc=unknown driver-impl-libvirt-xen=unknown driver-impl-vmware=missing driver-impl-hyperv=complete -driver-impl-ironic=missing +driver-impl-ironic=complete driver-impl-libvirt-vz-vm=missing driver-impl-libvirt-vz-ct=missing diff --git a/nova/conf/ironic.py b/nova/conf/ironic.py index a790106bec..58afc1b2bf 100644 --- a/nova/conf/ironic.py +++ b/nova/conf/ironic.py @@ -90,6 +90,12 @@ Related options: * api_max_retries """), + cfg.IntOpt( + 'serial_console_state_timeout', + default=10, + min=0, + help='Timeout (seconds) to wait for node serial console state ' + 'changed. Set to 0 to disable timeout.'), ] diff --git a/nova/conf/serial_console.py b/nova/conf/serial_console.py index d4c24fadc3..8552d84853 100644 --- a/nova/conf/serial_console.py +++ b/nova/conf/serial_console.py @@ -27,8 +27,8 @@ serial_opt_group = cfg.OptGroup("serial_console", title="The serial console feature", help=""" The serial console feature allows you to connect to a guest in case a -graphical console like VNC, RDP or SPICE is not available. This is only -supported for the libvirt driver.""") +graphical console like VNC, RDP or SPICE is not available. This is +supported for the libvirt and Ironic driver.""") enabled_opt = cfg.BoolOpt('enabled', default=False, diff --git a/nova/tests/unit/virt/ironic/test_driver.py b/nova/tests/unit/virt/ironic/test_driver.py index 477288e142..80a7167fb5 100644 --- a/nova/tests/unit/virt/ironic/test_driver.py +++ b/nova/tests/unit/virt/ironic/test_driver.py @@ -27,6 +27,7 @@ from nova.api.metadata import base as instance_metadata from nova.compute import power_state as nova_states from nova.compute import task_states from nova.compute import vm_states +from nova.console import type as console_type from nova import context as nova_context from nova import exception from nova import hash_ring @@ -34,6 +35,7 @@ from nova import objects from nova import servicegroup from nova import test from nova.tests.unit import fake_instance +from nova.tests.unit import matchers as nova_matchers from nova.tests.unit import utils from nova.tests.unit.virt.ironic import utils as ironic_utils from nova.virt import configdrive @@ -1918,3 +1920,311 @@ class NodeCacheTestCase(test.NoDBTestCase): expected_cache = {n.uuid: n for n in nodes[1:]} self.assertEqual(expected_cache, self.driver.node_cache) + + +@mock.patch.object(FAKE_CLIENT, 'node') +class IronicDriverConsoleTestCase(test.NoDBTestCase): + @mock.patch.object(cw, 'IronicClientWrapper', + lambda *_: FAKE_CLIENT_WRAPPER) + def setUp(self): + super(IronicDriverConsoleTestCase, self).setUp() + + self.driver = ironic_driver.IronicDriver(fake.FakeVirtAPI()) + self.ctx = nova_context.get_admin_context() + node_uuid = uuidutils.generate_uuid() + self.node = ironic_utils.get_test_node(driver='fake', uuid=node_uuid) + self.instance = fake_instance.fake_instance_obj(self.ctx, + node=node_uuid) + + # mock retries configs to avoid sleeps and make tests run quicker + CONF.set_default('api_max_retries', default=1, group='ironic') + CONF.set_default('api_retry_interval', default=0, group='ironic') + + self.stub_out('nova.virt.ironic.driver.IronicDriver.' + '_validate_instance_and_node', + lambda _, inst: self.node) + + def _create_console_data(self, enabled=True, console_type='socat', + url='tcp://127.0.0.1:10000'): + return { + 'console_enabled': enabled, + 'console_info': { + 'type': console_type, + 'url': url + } + } + + def test__get_node_console_with_reset_success(self, mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode']) + + def _fake_set_console_mode(node_uuid, mode): + # Set it up so that _fake_get_console() returns 'mode' + temp_data['target_mode'] = mode + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + + expected = self._create_console_data()['console_info'] + + result = self.driver._get_node_console_with_reset(self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertEqual(self.node.uuid, result['node'].uuid) + self.assertThat(result['console_info'], + nova_matchers.DictMatches(expected)) + + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test__get_node_console_with_reset_console_disabled(self, mock_log, + mock_node): + def _fake_log_debug(msg, *args, **kwargs): + regex = r'Console is disabled for instance .*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.return_value = \ + self._create_console_data(enabled=False) + mock_log.debug.side_effect = _fake_log_debug + + self.assertRaises(exception.ConsoleNotAvailable, + self.driver._get_node_console_with_reset, + self.instance) + + mock_node.get_console.assert_called_once_with(self.node.uuid) + mock_node.set_console_mode.assert_not_called() + self.assertTrue(mock_log.debug.called) + + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test__get_node_console_with_reset_set_mode_failed(self, mock_log, + mock_node): + def _fake_log_error(msg, *args, **kwargs): + regex = r'Failed to set console mode .*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.return_value = self._create_console_data() + mock_node.set_console_mode.side_effect = exception.NovaException() + mock_log.error.side_effect = _fake_log_error + + self.assertRaises(exception.ConsoleNotAvailable, + self.driver._get_node_console_with_reset, + self.instance) + + mock_node.get_console.assert_called_once_with(self.node.uuid) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertTrue(mock_log.error.called) + + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test__get_node_console_with_reset_wait_failed(self, mock_log, + mock_node): + def _fake_get_console(node_uuid): + if mock_node.set_console_mode.called: + # After the call to set_console_mode(), then _wait_state() + # will call _get_console() to check the result. + raise exception.NovaException() + else: + return self._create_console_data() + + def _fake_log_error(msg, *args, **kwargs): + regex = r'Failed to acquire console information for instance .*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.side_effect = _fake_get_console + mock_log.error.side_effect = _fake_log_error + + self.assertRaises(exception.ConsoleNotAvailable, + self.driver._get_node_console_with_reset, + self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertTrue(mock_log.error.called) + + @mock.patch.object(ironic_driver, '_CONSOLE_STATE_CHECKING_INTERVAL', 0.05) + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test__get_node_console_with_reset_wait_timeout(self, mock_log, + mock_node): + # Set timeout to a small value to reduce testing time + # Note: timeout value is integer, use enforce_type=False to set it + # to a floating number. + CONF.set_override('serial_console_state_timeout', 0.1, + group='ironic', enforce_type=False) + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode']) + + def _fake_set_console_mode(node_uuid, mode): + # This causes the _wait_state() will timeout because + # the target mode never gets set successfully. + temp_data['target_mode'] = not mode + + def _fake_log_error(msg, *args, **kwargs): + regex = r'Timeout while waiting for console mode to be set .*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + mock_log.error.side_effect = _fake_log_error + + self.assertRaises(exception.ConsoleNotAvailable, + self.driver._get_node_console_with_reset, + self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertTrue(mock_log.error.called) + + def test_get_serial_console_socat(self, mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode']) + + def _fake_set_console_mode(node_uuid, mode): + temp_data['target_mode'] = mode + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + + result = self.driver.get_serial_console(self.ctx, self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertIsInstance(result, console_type.ConsoleSerial) + self.assertEqual('127.0.0.1', result.host) + self.assertEqual(10000, result.port) + + def test_get_serial_console_socat_disabled(self, mock_node): + mock_node.get_console.return_value = \ + self._create_console_data(enabled=False) + + self.assertRaises(exception.ConsoleTypeUnavailable, + self.driver.get_serial_console, + self.ctx, self.instance) + mock_node.get_console.assert_called_once_with(self.node.uuid) + mock_node.set_console_mode.assert_not_called() + + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test_get_serial_console_socat_invalid_url(self, mock_log, mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode'], + url='an invalid url') + + def _fake_set_console_mode(node_uuid, mode): + temp_data['target_mode'] = mode + + def _fake_log_error(msg, *args, **kwargs): + regex = r'Invalid Socat console URL .*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + mock_log.error.side_effect = _fake_log_error + + self.assertRaises(exception.ConsoleTypeUnavailable, + self.driver.get_serial_console, + self.ctx, self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertTrue(mock_log.error.called) + + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test_get_serial_console_socat_invalid_url_2(self, mock_log, mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode'], + url='http://abcxyz:1a1b') + + def _fake_set_console_mode(node_uuid, mode): + temp_data['target_mode'] = mode + + def _fake_log_error(msg, *args, **kwargs): + regex = r'Invalid Socat console URL .*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + mock_log.error.side_effect = _fake_log_error + + self.assertRaises(exception.ConsoleTypeUnavailable, + self.driver.get_serial_console, + self.ctx, self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertTrue(mock_log.error.called) + + @mock.patch.object(ironic_driver, 'LOG', autospec=True) + def test_get_serial_console_socat_unsupported_scheme(self, mock_log, + mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode'], + url='ssl://127.0.0.1:10000') + + def _fake_set_console_mode(node_uuid, mode): + temp_data['target_mode'] = mode + + def _fake_log_warning(msg, *args, **kwargs): + regex = r'Socat serial console only supports \"tcp\".*' + self.assertThat(msg, matchers.MatchesRegex(regex)) + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + mock_log.warning.side_effect = _fake_log_warning + + self.assertRaises(exception.ConsoleTypeUnavailable, + self.driver.get_serial_console, + self.ctx, self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertTrue(mock_log.warning.called) + + def test_get_serial_console_socat_tcp6(self, mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode'], + url='tcp://[::1]:10000') + + def _fake_set_console_mode(node_uuid, mode): + temp_data['target_mode'] = mode + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + + result = self.driver.get_serial_console(self.ctx, self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) + self.assertIsInstance(result, console_type.ConsoleSerial) + self.assertEqual('::1', result.host) + self.assertEqual(10000, result.port) + + def test_get_serial_console_shellinabox(self, mock_node): + temp_data = {'target_mode': True} + + def _fake_get_console(node_uuid): + return self._create_console_data(enabled=temp_data['target_mode'], + console_type='shellinabox') + + def _fake_set_console_mode(node_uuid, mode): + temp_data['target_mode'] = mode + + mock_node.get_console.side_effect = _fake_get_console + mock_node.set_console_mode.side_effect = _fake_set_console_mode + + self.assertRaises(exception.ConsoleTypeUnavailable, + self.driver.get_serial_console, + self.ctx, self.instance) + + self.assertGreater(mock_node.get_console.call_count, 1) + self.assertEqual(2, mock_node.set_console_mode.call_count) diff --git a/nova/virt/ironic/driver.py b/nova/virt/ironic/driver.py index 5ef3394d9e..019c6159a8 100644 --- a/nova/virt/ironic/driver.py +++ b/nova/virt/ironic/driver.py @@ -31,6 +31,7 @@ from oslo_service import loopingcall from oslo_utils import excutils from oslo_utils import importutils import six +import six.moves.urllib.parse as urlparse from nova.api.metadata import base as instance_metadata from nova.compute import arch @@ -40,6 +41,7 @@ from nova.compute import task_states from nova.compute import vm_mode from nova.compute import vm_states import nova.conf +from nova.console import type as console_type from nova import context as nova_context from nova import exception from nova import hash_ring @@ -79,6 +81,9 @@ _NODE_FIELDS = ('uuid', 'power_state', 'target_power_state', 'provision_state', 'target_provision_state', 'last_error', 'maintenance', 'properties', 'instance_uuid') +# Console state checking interval in seconds +_CONSOLE_STATE_CHECKING_INTERVAL = 1 + def map_power_state(state): try: @@ -1225,3 +1230,154 @@ class IronicDriver(virt_driver.ComputeDriver): # flat network, go ahead and allow the port to be bound return super(IronicDriver, self).network_binding_host_id( context, instance) + + def _get_node_console_with_reset(self, instance): + """Acquire console information for an instance. + + If the console is enabled, the console will be re-enabled + before returning. + + :param instance: nova instance + :return: a dictionary with below values + { 'node': ironic node + 'console_info': node console info } + :raise ConsoleNotAvailable: if console is unavailable + for the instance + """ + node = self._validate_instance_and_node(instance) + node_uuid = node.uuid + + def _get_console(): + """Request ironicclient to acquire node console.""" + try: + return self.ironicclient.call('node.get_console', node_uuid) + except (exception.NovaException, # Retry failed + ironic.exc.InternalServerError, # Validations + ironic.exc.BadRequest) as e: # Maintenance + LOG.error(_LE('Failed to acquire console information for ' + 'instance %(inst)s: %(reason)s'), + {'inst': instance.uuid, + 'reason': e}) + raise exception.ConsoleNotAvailable() + + def _wait_state(state): + """Wait for the expected console mode to be set on node.""" + console = _get_console() + if console['console_enabled'] == state: + raise loopingcall.LoopingCallDone(retvalue=console) + + _log_ironic_polling('set console mode', node, instance) + + # Return False to start backing off + return False + + def _enable_console(mode): + """Request ironicclient to enable/disable node console.""" + try: + self.ironicclient.call('node.set_console_mode', node_uuid, + mode) + except (exception.NovaException, # Retry failed + ironic.exc.InternalServerError, # Validations + ironic.exc.BadRequest) as e: # Maintenance + LOG.error(_LE('Failed to set console mode to "%(mode)s" ' + 'for instance %(inst)s: %(reason)s'), + {'mode': mode, + 'inst': instance.uuid, + 'reason': e}) + raise exception.ConsoleNotAvailable() + + # Waiting for the console state to change (disabled/enabled) + try: + timer = loopingcall.BackOffLoopingCall(_wait_state, state=mode) + return timer.start( + starting_interval=_CONSOLE_STATE_CHECKING_INTERVAL, + timeout=CONF.ironic.serial_console_state_timeout, + jitter=0.5).wait() + except loopingcall.LoopingCallTimeOut: + LOG.error(_LE('Timeout while waiting for console mode to be ' + 'set to "%(mode)s" on node %(node)s'), + {'mode': mode, + 'node': node_uuid}) + raise exception.ConsoleNotAvailable() + + # Acquire the console + console = _get_console() + + # NOTE: Resetting console is a workaround to force acquiring + # console when it has already been acquired by another user/operator. + # IPMI serial console does not support multi session, so + # resetting console will deactivate any active one without + # warning the operator. + if console['console_enabled']: + try: + # Disable console + _enable_console(False) + # Then re-enable it + console = _enable_console(True) + except exception.ConsoleNotAvailable: + # NOTE: We try to do recover on failure. + # But if recover fails, the console may remain in + # "disabled" state and cause any new connection + # will be refused. + console = _enable_console(True) + + if console['console_enabled']: + return {'node': node, + 'console_info': console['console_info']} + else: + LOG.debug('Console is disabled for instance %s', + instance.uuid) + raise exception.ConsoleNotAvailable() + + def get_serial_console(self, context, instance): + """Acquire serial console information. + + :param context: request context + :param instance: nova instance + :return: ConsoleSerial object + :raise ConsoleTypeUnavailable: if serial console is unavailable + for the instance + """ + LOG.debug('Getting serial console', instance=instance) + try: + result = self._get_node_console_with_reset(instance) + except exception.ConsoleNotAvailable: + raise exception.ConsoleTypeUnavailable(console_type='serial') + + node = result['node'] + console_info = result['console_info'] + + if console_info["type"] != "socat": + LOG.warning(_LW('Console type "%(type)s" (of ironic node ' + '%(node)s) does not support Nova serial console'), + {'type': console_info["type"], + 'node': node.uuid}, + instance=instance) + raise exception.ConsoleTypeUnavailable(console_type='serial') + + # Parse and check the console url + url = urlparse.urlparse(console_info["url"]) + try: + scheme = url.scheme + hostname = url.hostname + port = url.port + if not (scheme and hostname and port): + raise AssertionError() + except (ValueError, AssertionError): + LOG.error(_LE('Invalid Socat console URL "%(url)s" ' + '(ironic node %(node)s)'), + {'url': console_info["url"], + 'node': node.uuid}, + instance=instance) + raise exception.ConsoleTypeUnavailable(console_type='serial') + + if scheme == "tcp": + return console_type.ConsoleSerial(host=hostname, + port=port) + else: + LOG.warning(_LW('Socat serial console only supports "tcp". ' + 'This URL is "%(url)s" (ironic node %(node)s).'), + {'url': console_info["url"], + 'node': node.uuid}, + instance=instance) + raise exception.ConsoleTypeUnavailable(console_type='serial') diff --git a/releasenotes/notes/ironic-serial-console-support-82632bd4db6d1fda.yaml b/releasenotes/notes/ironic-serial-console-support-82632bd4db6d1fda.yaml new file mode 100644 index 0000000000..076767e1bc --- /dev/null +++ b/releasenotes/notes/ironic-serial-console-support-82632bd4db6d1fda.yaml @@ -0,0 +1,9 @@ +--- +features: + - Adds serial console support to Ironic driver. Nova now supports + serial console to Ironic bare metals for Ironic ``socat`` console + driver. + In order to use this feature, serial console must be configured in Nova + and the Ironic ``socat`` console driver must be used and configured + in Ironic. Ironic serial console configuration is documented in + http://docs.openstack.org/developer/ironic/deploy/console.html.