nova-status: add basic placement status checking
This adds validation that the placement api is in working order, and handles all the errors we're currently aware of and can anticipate. Part of blueprint resource-providers-scheduler-db-filters Change-Id: I6c71ef09ce44dee0ac3a80eb06a37234e45e0259
This commit is contained in:
Sean Dague
committed by
Matt Riedemann
Matt Riedemann
parent
4103f5e462
commit
5978013ca2
+56
-4
@@ -24,6 +24,9 @@ import traceback
|
||||
|
||||
# enum comes from the enum34 package if python < 3.4, else it's stdlib
|
||||
import enum
|
||||
from keystoneauth1 import exceptions as ks_exc
|
||||
from keystoneauth1 import loading as keystone
|
||||
from keystoneauth1 import session
|
||||
from oslo_config import cfg
|
||||
import prettytable
|
||||
from sqlalchemy import func as sqlfunc
|
||||
@@ -150,11 +153,60 @@ class UpgradeCommands(object):
|
||||
|
||||
return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
|
||||
|
||||
def _placement_get(self, path):
|
||||
"""Do an HTTP get call against placement engine.
|
||||
|
||||
This is in a dedicated method to make it easier for unit
|
||||
testing purposes.
|
||||
|
||||
"""
|
||||
ks_filter = {'service_type': 'placement',
|
||||
'region_name': CONF.placement.os_region_name}
|
||||
auth = keystone.load_auth_from_conf_options(
|
||||
CONF, 'placement')
|
||||
client = session.Session(auth=auth)
|
||||
return client.get(path, endpoint_filter=ks_filter).json()
|
||||
|
||||
def _check_placement(self):
|
||||
# TODO(mriedem): check to see that the placement API service is running
|
||||
# and we can connect to it, and that the number of resource providers
|
||||
# in the placement service is greater than or equal to the number of
|
||||
# compute nodes in the database.
|
||||
"""Checks to see if the placement API is ready for scheduling.
|
||||
|
||||
Checks to see that the placement API service is registered in the
|
||||
service catalog and that we can make requests against it. Also checks
|
||||
that there are compute nodes registered with the placement service
|
||||
as resource providers so that when the Ocata nova-scheduler code starts
|
||||
handling requests it has somewhere to send those builds.
|
||||
"""
|
||||
try:
|
||||
versions = self._placement_get("/")
|
||||
max_version = float(versions["versions"][0]["max_version"])
|
||||
# The required version is a bit tricky but we know that we at least
|
||||
# need 1.0 for Newton computes. This minimum might change in the
|
||||
# future.
|
||||
needs_version = 1.0
|
||||
if max_version < needs_version:
|
||||
msg = (_('Placement API version %(needed)s needed, '
|
||||
'you have %(current)s.') %
|
||||
{'needed': needs_version, 'current': max_version})
|
||||
return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
|
||||
except ks_exc.MissingAuthPlugin:
|
||||
msg = _('No credentials specified for placement API in nova.conf.')
|
||||
return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
|
||||
except ks_exc.Unauthorized:
|
||||
msg = _('Placement service credentials do not work.')
|
||||
return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
|
||||
except ks_exc.EndpointNotFound:
|
||||
msg = _('Placement API endpoint not found.')
|
||||
return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
|
||||
except ks_exc.NotFound:
|
||||
msg = _('Placement API does not seem to be running.')
|
||||
return UpgradeCheckResult(UpgradeCheckCode.FAILURE, msg)
|
||||
|
||||
# TODO(mriedem): The placement service is running, fun! Now let's query
|
||||
# the API DB to count the number of resource providers and compare that
|
||||
# to the number of compute nodes (probably across all cells?). If there
|
||||
# are no resource providers it's a clear fail. If there are fewer RPs
|
||||
# than computes then it's a warning because you might be underutilized.
|
||||
|
||||
return UpgradeCheckResult(UpgradeCheckCode.SUCCESS)
|
||||
|
||||
# The format of the check functions is to return an UpgradeCheckResult
|
||||
|
||||
@@ -20,6 +20,9 @@ import fixtures
|
||||
import mock
|
||||
from six.moves import StringIO
|
||||
|
||||
from keystoneauth1 import exceptions as ks_exc
|
||||
from keystoneauth1 import loading as keystone
|
||||
|
||||
from nova.cmd import status
|
||||
import nova.conf
|
||||
from nova import context
|
||||
@@ -87,6 +90,105 @@ class TestNovaStatusMain(test.NoDBTestCase):
|
||||
self.assertIn('wut', output)
|
||||
|
||||
|
||||
class TestPlacementCheck(test.NoDBTestCase):
|
||||
"""Tests the nova-status placement checks.
|
||||
|
||||
These are done with mock as the ability to replicate all failure
|
||||
domains otherwise is quite complicated. Using a devstack
|
||||
environment you can validate each of these tests are matching
|
||||
reality.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestPlacementCheck, self).setUp()
|
||||
self.output = StringIO()
|
||||
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.output))
|
||||
self.cmd = status.UpgradeCommands()
|
||||
|
||||
@mock.patch.object(keystone, "load_auth_from_conf_options")
|
||||
def test_no_auth(self, auth):
|
||||
"""Test failure when no credentials are specified.
|
||||
|
||||
Replicate in devstack: start devstack with or without
|
||||
placement engine, remove the auth section from the [placement]
|
||||
block in nova.conf.
|
||||
"""
|
||||
auth.side_effect = ks_exc.MissingAuthPlugin()
|
||||
res = self.cmd._check_placement()
|
||||
self.assertEqual(status.UpgradeCheckCode.FAILURE, res.code)
|
||||
self.assertIn('No credentials specified', res.details)
|
||||
|
||||
@mock.patch.object(status.UpgradeCommands, "_placement_get")
|
||||
def test_invalid_auth(self, get):
|
||||
"""Test failure when wrong credentials are specified or service user
|
||||
doesn't exist.
|
||||
|
||||
Replicate in devstack: start devstack with or without
|
||||
placement engine, specify random credentials in auth section
|
||||
from the [placement] block in nova.conf.
|
||||
|
||||
"""
|
||||
get.side_effect = ks_exc.Unauthorized()
|
||||
res = self.cmd._check_placement()
|
||||
self.assertEqual(status.UpgradeCheckCode.FAILURE, res.code)
|
||||
self.assertIn('Placement service credentials do not work', res.details)
|
||||
|
||||
@mock.patch.object(status.UpgradeCommands, "_placement_get")
|
||||
def test_invalid_endpoint(self, get):
|
||||
"""Test failure when no endpoint exists.
|
||||
|
||||
Replicate in devstack: start devstack without placement
|
||||
engine, but create valid placement service user and specify it
|
||||
in auth section of [placement] in nova.conf.
|
||||
"""
|
||||
get.side_effect = ks_exc.EndpointNotFound()
|
||||
res = self.cmd._check_placement()
|
||||
self.assertEqual(status.UpgradeCheckCode.FAILURE, res.code)
|
||||
self.assertIn('Placement API endpoint not found', res.details)
|
||||
|
||||
@mock.patch.object(status.UpgradeCommands, "_placement_get")
|
||||
def test_down_endpoint(self, get):
|
||||
"""Test failure when endpoint is down.
|
||||
|
||||
Replicate in devstack: start devstack with placement
|
||||
engine, disable placement engine apache config.
|
||||
"""
|
||||
get.side_effect = ks_exc.NotFound()
|
||||
res = self.cmd._check_placement()
|
||||
self.assertEqual(status.UpgradeCheckCode.FAILURE, res.code)
|
||||
self.assertIn('Placement API does not seem to be running', res.details)
|
||||
|
||||
@mock.patch.object(status.UpgradeCommands, "_placement_get")
|
||||
def test_valid_version(self, get):
|
||||
get.return_value = {
|
||||
"versions": [
|
||||
{
|
||||
"min_version": "1.0",
|
||||
"max_version": "1.0",
|
||||
"id": "v1.0"
|
||||
}
|
||||
]
|
||||
}
|
||||
res = self.cmd._check_placement()
|
||||
self.assertEqual(status.UpgradeCheckCode.SUCCESS, res.code)
|
||||
|
||||
@mock.patch.object(status.UpgradeCommands, "_placement_get")
|
||||
def test_invalid_version(self, get):
|
||||
get.return_value = {
|
||||
"versions": [
|
||||
{
|
||||
"min_version": "0.9",
|
||||
"max_version": "0.9",
|
||||
"id": "v1.0"
|
||||
}
|
||||
]
|
||||
}
|
||||
res = self.cmd._check_placement()
|
||||
self.assertEqual(status.UpgradeCheckCode.FAILURE, res.code)
|
||||
self.assertIn('Placement API version 1.0 needed, you have 0.9',
|
||||
res.details)
|
||||
|
||||
|
||||
class TestUpgradeCheckBasic(test.NoDBTestCase):
|
||||
"""Tests for the nova-status upgrade check command.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user