From 2337fab0979b72bbc7e7730e94518a0e835a2751 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Mon, 25 Oct 2010 03:45:19 +0900 Subject: [PATCH 001/119] part way through porting the codebase off of twisted this provides a very basic eventlet-based service replacement for the twistd-based services, a replacement for task.LoopingCall also adds nova-combined with the goal of running a single service when doing local testing and dev --- bin/nova-combined | 61 ++++++++ bin/nova-compute | 12 +- bin/nova-network | 14 +- nova/compute/disk.py | 66 ++++----- nova/compute/manager.py | 28 ++-- nova/flags.py | 7 + nova/manager.py | 1 - nova/network/manager.py | 4 +- nova/server.py | 6 +- nova/service_eventlet.py | 288 ++++++++++++++++++++++++++++++++++++++ nova/utils.py | 36 +++++ nova/virt/fake.py | 6 +- nova/virt/libvirt_conn.py | 176 +++++++++++------------ nova/virt/xenapi.py | 103 ++++++-------- 14 files changed, 576 insertions(+), 232 deletions(-) create mode 100755 bin/nova-combined create mode 100644 nova/service_eventlet.py diff --git a/bin/nova-combined b/bin/nova-combined new file mode 100755 index 0000000000..65865acd98 --- /dev/null +++ b/bin/nova-combined @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" + Twistd daemon for the nova compute nodes. +""" + +import eventlet +eventlet.monkey_patch() + +import os +import sys + +from eventlet import greenthread + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from nova import api +from nova import flags +from nova import service_eventlet +from nova import wsgi + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('api_port', 8773, 'API port') + + +if __name__ == '__main__': + FLAGS(sys.argv) + + compute = service_eventlet.Service.create(binary='nova-compute') + network = service_eventlet.Service.create(binary='nova-network') + volume = service_eventlet.Service.create(binary='nova-volume') + scheduler = service_eventlet.Service.create(binary='nova-scheduler') + #objectstore = service_eventlet.Service.create(binary='nova-objectstore') + + service_eventlet.serve(compute, network, volume, scheduler) + wsgi.run_server(api.API(), FLAGS.api_port) + diff --git a/bin/nova-compute b/bin/nova-compute index 1724e96594..600fbb8973 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -21,6 +21,9 @@ Twistd daemon for the nova compute nodes. """ +import eventlet +eventlet.monkey_patch() + import os import sys @@ -32,12 +35,7 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service -from nova import twistd - +from nova import service_eventlet if __name__ == '__main__': - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() # pylint: disable=C0103 + service_eventlet.serve() diff --git a/bin/nova-network b/bin/nova-network index fa88aeb476..600fbb8973 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -18,9 +18,12 @@ # under the License. """ - Twistd daemon for the nova network nodes. + Twistd daemon for the nova compute nodes. """ +import eventlet +eventlet.monkey_patch() + import os import sys @@ -32,12 +35,7 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service -from nova import twistd - +from nova import service_eventlet if __name__ == '__main__': - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() # pylint: disable-msg=C0103 + service_eventlet.serve() diff --git a/nova/compute/disk.py b/nova/compute/disk.py index e362b4507e..ad4c2c0925 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -25,8 +25,6 @@ import logging import os import tempfile -from twisted.internet import defer - from nova import exception from nova import flags @@ -38,7 +36,6 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') -@defer.inlineCallbacks def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2', execute=None): """Takes a single partition represented by infile and writes a bootable @@ -60,10 +57,10 @@ def partition(infile, outfile, local_bytes=0, resize=True, file_size = os.path.getsize(infile) if resize and file_size < FLAGS.minimum_root_size: last_sector = FLAGS.minimum_root_size / sector_size - 1 - yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (infile, last_sector, sector_size)) - yield execute('e2fsck -fp %s' % infile, check_exit_code=False) - yield execute('resize2fs %s' % infile) + execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (infile, last_sector, sector_size)) + execute('e2fsck -fp %s' % infile, check_exit_code=False) + execute('resize2fs %s' % infile) file_size = FLAGS.minimum_root_size elif file_size % sector_size != 0: logging.warn("Input partition size not evenly divisible by" @@ -82,30 +79,29 @@ def partition(infile, outfile, local_bytes=0, resize=True, last_sector = local_last # e # create an empty file - yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, mbr_last, sector_size)) + execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (outfile, mbr_last, sector_size)) # make mbr partition - yield execute('parted --script %s mklabel msdos' % outfile) + execute('parted --script %s mklabel msdos' % outfile) # append primary file - yield execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' - % (infile, outfile, FLAGS.block_size)) + execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' + % (infile, outfile, FLAGS.block_size)) # make primary partition - yield execute('parted --script %s mkpart primary %ds %ds' - % (outfile, primary_first, primary_last)) + execute('parted --script %s mkpart primary %ds %ds' + % (outfile, primary_first, primary_last)) if local_bytes > 0: # make the file bigger - yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, last_sector, sector_size)) + execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (outfile, last_sector, sector_size)) # make and format local partition - yield execute('parted --script %s mkpartfs primary %s %ds %ds' - % (outfile, local_type, local_first, local_last)) + execute('parted --script %s mkpartfs primary %s %ds %ds' + % (outfile, local_type, local_first, local_last)) -@defer.inlineCallbacks def inject_data(image, key=None, net=None, partition=None, execute=None): """Injects a ssh key and optionally net data into a disk image. @@ -115,26 +111,26 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): If partition is not specified it mounts the image as a single partition. """ - out, err = yield execute('sudo losetup -f --show %s' % image) + out, err = execute('sudo losetup -f --show %s' % image) if err: raise exception.Error('Could not attach image to loopback: %s' % err) device = out.strip() try: if not partition is None: # create partition - out, err = yield execute('sudo kpartx -a %s' % device) + out, err = execute('sudo kpartx -a %s' % device) if err: raise exception.Error('Failed to load partition: %s' % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], partition) else: mapped_device = device - out, err = yield execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + out, err = execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir - out, err = yield execute( + out, err = execute( 'sudo mount %s %s' % (mapped_device, tmpdir)) if err: raise exception.Error('Failed to mount filesystem: %s' % err) @@ -142,35 +138,33 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): try: if key: # inject key file - yield _inject_key_into_fs(key, tmpdir, execute=execute) + _inject_key_into_fs(key, tmpdir, execute=execute) if net: - yield _inject_net_into_fs(net, tmpdir, execute=execute) + _inject_net_into_fs(net, tmpdir, execute=execute) finally: # unmount device - yield execute('sudo umount %s' % mapped_device) + execute('sudo umount %s' % mapped_device) finally: # remove temporary directory - yield execute('rmdir %s' % tmpdir) + execute('rmdir %s' % tmpdir) if not partition is None: # remove partitions - yield execute('sudo kpartx -d %s' % device) + execute('sudo kpartx -d %s' % device) finally: # remove loopback - yield execute('sudo losetup -d %s' % device) + execute('sudo losetup -d %s' % device) -@defer.inlineCallbacks def _inject_key_into_fs(key, fs, execute=None): sshdir = os.path.join(os.path.join(fs, 'root'), '.ssh') - yield execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - yield execute('sudo chown root %s' % sshdir) - yield execute('sudo chmod 700 %s' % sshdir) + execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter + execute('sudo chown root %s' % sshdir) + execute('sudo chmod 700 %s' % sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - yield execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') -@defer.inlineCallbacks def _inject_net_into_fs(net, fs, execute=None): netfile = os.path.join(os.path.join(os.path.join( fs, 'etc'), 'network'), 'interfaces') - yield execute('sudo tee %s' % netfile, net) + execute('sudo tee %s' % netfile, net) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 523bb88936..a105a1dd03 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -25,8 +25,6 @@ import datetime import logging import os -from twisted.internet import defer - from nova import exception from nova import flags from nova import manager @@ -54,7 +52,7 @@ class ComputeManager(manager.Manager): self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(*args, **kwargs) - + def _update_state(self, context, instance_id): """Update the state of an instance from the driver info""" # FIXME(ja): include other fields from state? @@ -62,12 +60,10 @@ class ComputeManager(manager.Manager): state = self.driver.get_info(instance_ref.name)['state'] self.db.instance_set_state(context, instance_id, state) - @defer.inlineCallbacks @exception.wrap_exception def refresh_security_group(self, context, security_group_id, **_kwargs): - yield self.driver.refresh_security_group(security_group_id) + self.driver.refresh_security_group(security_group_id) - @defer.inlineCallbacks @exception.wrap_exception def run_instance(self, context, instance_id, **_kwargs): """Launch a new instance with specified options.""" @@ -89,7 +85,7 @@ class ComputeManager(manager.Manager): 'spawning') try: - yield self.driver.spawn(instance_ref) + self.driver.spawn(instance_ref) now = datetime.datetime.utcnow() self.db.instance_update(context, instance_id, @@ -103,7 +99,6 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) - @defer.inlineCallbacks @exception.wrap_exception def terminate_instance(self, context, instance_id): """Terminate an instance on this machine.""" @@ -116,12 +111,11 @@ class ComputeManager(manager.Manager): raise exception.Error('trying to destroy already destroyed' ' instance: %s' % instance_id) - yield self.driver.destroy(instance_ref) + self.driver.destroy(instance_ref) # TODO(ja): should we keep it in a terminated state for a bit? self.db.instance_destroy(context, instance_id) - @defer.inlineCallbacks @exception.wrap_exception def reboot_instance(self, context, instance_id): """Reboot an instance on this server.""" @@ -142,7 +136,7 @@ class ComputeManager(manager.Manager): instance_id, power_state.NOSTATE, 'rebooting') - yield self.driver.reboot(instance_ref) + self.driver.reboot(instance_ref) self._update_state(context, instance_id) @exception.wrap_exception @@ -154,7 +148,6 @@ class ComputeManager(manager.Manager): return self.driver.get_console_output(instance_ref) - @defer.inlineCallbacks @exception.wrap_exception def attach_volume(self, context, instance_id, volume_id, mountpoint): """Attach a volume to an instance.""" @@ -164,13 +157,12 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) dev_path = yield self.volume_manager.setup_compute_volume(context, volume_id) - yield self.driver.attach_volume(instance_ref['ec2_id'], - dev_path, - mountpoint) + self.driver.attach_volume(instance_ref['ec2_id'], + dev_path, + mountpoint) self.db.volume_attached(context, volume_id, instance_id, mountpoint) defer.returnValue(True) - @defer.inlineCallbacks @exception.wrap_exception def detach_volume(self, context, instance_id, volume_id): """Detach a volume from an instance.""" @@ -180,7 +172,7 @@ class ComputeManager(manager.Manager): volume_id) instance_ref = self.db.instance_get(context, instance_id) volume_ref = self.db.volume_get(context, volume_id) - yield self.driver.detach_volume(instance_ref['ec2_id'], - volume_ref['mountpoint']) + self.driver.detach_volume(instance_ref['ec2_id'], + volume_ref['mountpoint']) self.db.volume_detached(context, volume_id) defer.returnValue(True) diff --git a/nova/flags.py b/nova/flags.py index f3b0384ad8..da9987700d 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -139,6 +139,8 @@ class FlagValues(gflags.FlagValues): FLAGS = FlagValues() +gflags.FLAGS = FLAGS + def _wrapper(func): def _wrapped(*args, **kw): @@ -159,6 +161,11 @@ DEFINE_list = _wrapper(gflags.DEFINE_list) DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) +DEFINE_flag = _wrapper(gflags.DEFINE_flag) + +HelpFlag = gflags.HelpFlag +HelpshortFlag = gflags.HelpshortFlag +HelpXMLFlag = gflags.HelpXMLFlag def DECLARE(name, module_string, flag_values=FLAGS): diff --git a/nova/manager.py b/nova/manager.py index 4244b2db4e..994d6e7af5 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -39,7 +39,6 @@ class Manager(object): db_driver = FLAGS.db_driver self.db = utils.import_object(db_driver) # pylint: disable-msg=C0103 - @defer.inlineCallbacks def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval""" yield diff --git a/nova/network/manager.py b/nova/network/manager.py index fddb77663a..37a15ae050 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -25,7 +25,6 @@ import logging import math import IPy -from twisted.internet import defer from nova import context from nova import db @@ -315,10 +314,9 @@ class FlatDHCPManager(NetworkManager): class VlanManager(NetworkManager): """Vlan network with dhcp""" - @defer.inlineCallbacks def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval""" - yield super(VlanManager, self).periodic_tasks(context) + super(VlanManager, self).periodic_tasks(context) now = datetime.datetime.utcnow() timeout = FLAGS.fixed_ip_disassociate_timeout time = now - datetime.timedelta(seconds=timeout) diff --git a/nova/server.py b/nova/server.py index cb424caa19..fba340a54d 100644 --- a/nova/server.py +++ b/nova/server.py @@ -135,9 +135,9 @@ def daemonize(args, name, main): with daemon.DaemonContext( detach_process=FLAGS.daemonize, working_directory=FLAGS.working_directory, - pidfile=pidlockfile.TimeoutPIDLockFile(FLAGS.pidfile, - acquire_timeout=1, - threaded=False), + #pidfile=pidlockfile.TimeoutPIDLockFile(FLAGS.pidfile, + # acquire_timeout=1, + # threaded=False), stdin=stdin, stdout=stdout, stderr=stderr, diff --git a/nova/service_eventlet.py b/nova/service_eventlet.py new file mode 100644 index 0000000000..eac45a9810 --- /dev/null +++ b/nova/service_eventlet.py @@ -0,0 +1,288 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Generic Node baseclass for all workers that run on hosts +""" + +import inspect +import logging +import os +import sys + +from eventlet import event +from eventlet import greenthread +from eventlet import greenpool + +from nova import context +from nova import db +from nova import exception +from nova import flags +from nova import rpc +from nova import utils + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('report_interval', 10, + 'seconds between nodes reporting state to datastore', + lower_bound=1) + +flags.DEFINE_integer('periodic_interval', 60, + 'seconds between running periodic tasks', + lower_bound=1) + +flags.DEFINE_string('pidfile', None, + 'pidfile to use for this service') + + +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + + +class Service(object): + """Base class for workers that run on hosts.""" + + def __init__(self, host, binary, topic, manager, report_interval=None, + periodic_interval=None, *args, **kwargs): + self.host = host + self.binary = binary + self.topic = topic + self.manager_class_name = manager + self.report_interval = report_interval + self.periodic_interval = periodic_interval + super(Service, self).__init__(*args, **kwargs) + self.saved_args, self.saved_kwargs = args, kwargs + + def start(self): + manager_class = utils.import_class(self.manager_class_name) + self.manager = manager_class(host=self.host, *self.saved_args, + **self.saved_kwargs) + self.manager.init_host() + self.model_disconnected = False + ctxt = context.get_admin_context() + try: + service_ref = db.service_get_by_args(ctxt, + self.host, + self.binary) + self.service_id = service_ref['id'] + except exception.NotFound: + self._create_service_ref(ctxt) + + conn1 = rpc.Connection.instance(new=True) + conn2 = rpc.Connection.instance(new=True) + if self.report_interval: + consumer_all = rpc.AdapterConsumer( + connection=conn1, + topic=self.topic, + proxy=self) + consumer_node = rpc.AdapterConsumer( + connection=conn2, + topic='%s.%s' % (self.topic, self.host), + proxy=self) + + consumer_all.attach_to_eventlet() + consumer_node.attach_to_eventlet() + + pulse = utils.LoopingCall(self.report_state) + pulse.start(interval=self.report_interval, now=False) + + if self.periodic_interval: + pulse = utils.LoopingCall(self.periodic_tasks) + pulse.start(interval=self.periodic_interval, now=False) + + def _create_service_ref(self, context): + service_ref = db.service_create(context, + {'host': self.host, + 'binary': self.binary, + 'topic': self.topic, + 'report_count': 0}) + self.service_id = service_ref['id'] + + def __getattr__(self, key): + manager = self.__dict__.get('manager', None) + return getattr(manager, key) + + @classmethod + def create(cls, + host=None, + binary=None, + topic=None, + manager=None, + report_interval=None, + periodic_interval=None): + """Instantiates class and passes back application object. + + Args: + host, defaults to FLAGS.host + binary, defaults to basename of executable + topic, defaults to bin_name - "nova-" part + manager, defaults to FLAGS._manager + report_interval, defaults to FLAGS.report_interval + periodic_interval, defaults to FLAGS.periodic_interval + """ + if not host: + host = FLAGS.host + if not binary: + binary = os.path.basename(inspect.stack()[-1][1]) + if not topic: + topic = binary.rpartition("nova-")[2] + if not manager: + manager = FLAGS.get('%s_manager' % topic, None) + if not report_interval: + report_interval = FLAGS.report_interval + if not periodic_interval: + periodic_interval = FLAGS.periodic_interval + logging.warn("Starting %s node", topic) + service_obj = cls(host, binary, topic, manager, + report_interval, periodic_interval) + + return service_obj + + def kill(self): + """Destroy the service object in the datastore""" + try: + db.service_destroy(context.get_admin_context(), self.service_id) + except exception.NotFound: + logging.warn("Service killed that has no database entry") + + def periodic_tasks(self): + """Tasks to be run at a periodic interval""" + self.manager.periodic_tasks(context.get_admin_context()) + + def report_state(self): + """Update the state of this service in the datastore.""" + ctxt = context.get_admin_context() + try: + try: + service_ref = db.service_get(ctxt, self.service_id) + except exception.NotFound: + logging.debug("The service database object disappeared, " + "Recreating it.") + self._create_service_ref(ctxt) + service_ref = db.service_get(ctxt, self.service_id) + + db.service_update(ctxt, + self.service_id, + {'report_count': service_ref['report_count'] + 1}) + + # TODO(termie): make this pattern be more elegant. + if getattr(self, "model_disconnected", False): + self.model_disconnected = False + logging.error("Recovered model server connection!") + + # TODO(vish): this should probably only catch connection errors + except Exception: # pylint: disable-msg=W0702 + if not getattr(self, "model_disconnected", False): + self.model_disconnected = True + logging.exception("model server went away") + + +def stop(pidfile): + """ + Stop the daemon + """ + # Get the pid from the pidfile + try: + pf = file(pidfile, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + pid = None + + if not pid: + message = "pidfile %s does not exist. Daemon not running?\n" + sys.stderr.write(message % pidfile) + # Not an error in a restart + return + + # Try killing the daemon process + try: + while 1: + os.kill(pid, signal.SIGKILL) + time.sleep(0.1) + except OSError, err: + err = str(err) + if err.find("No such process") > 0: + if os.path.exists(pidfile): + os.remove(pidfile) + else: + print str(err) + sys.exit(1) + + +def serve(*services): + argv = FLAGS(sys.argv) + + if not services: + services = [Service.create()] + + name = '_'.join(x.binary for x in services) + logging.debug("Serving %s" % name) + + logging.getLogger('amqplib').setLevel(logging.DEBUG) + + if not FLAGS.pidfile: + FLAGS.pidfile = '%s.pid' % name + # NOTE(vish): if we're running nodaemon, redirect the log to stdout + #if FLAGS.nodaemon and not FLAGS.logfile: + # FLAGS.logfile = "-" + #if not FLAGS.logfile: + # FLAGS.logfile = '%s.log' % name + #if not FLAGS.prefix: + # FLAGS.prefix = name + #elif FLAGS.prefix.endswith('twisted'): + # FLAGS.prefix = FLAGS.prefix.replace('twisted', name) + + action = 'start' + if len(argv) > 1: + action = argv.pop() + + if action == 'stop': + stop(FLAGS.pidfile) + sys.exit() + elif action == 'restart': + stop(FLAGS.pidfile) + elif action == 'start': + pass + else: + print 'usage: %s [options] [start|stop|restart]' % argv[0] + sys.exit(1) + + #formatter = logging.Formatter( + # '(%(name)s): %(levelname)s %(message)s') + #handler = logging.StreamHandler() + #handler.setFormatter(formatter) + #logging.getLogger().addHandler(handler) + + if FLAGS.verbose: + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.getLogger().setLevel(logging.WARNING) + + logging.debug("Full set of FLAGS:") + for flag in FLAGS: + logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) + + for x in services: + x.start() + + #while True: + # greenthread.sleep(5) + + diff --git a/nova/utils.py b/nova/utils.py index 7683fc9f45..a219e47bf8 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -29,6 +29,9 @@ import subprocess import socket import sys +from eventlet import event +from eventlet import greenthread + from twisted.internet.threads import deferToThread from nova import exception @@ -212,3 +215,36 @@ def deferredToThread(f): def g(*args, **kwargs): return deferToThread(f, *args, **kwargs) return g + + +class LoopingCall(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + + def start(self, interval, now=True): + self._running = True + done = event.Event() + def _inner(): + if not now: + greenthread.sleep(interval) + try: + while self._running: + self.f(*self.args, **self.kw) + greenthread.sleep(interval) + except Exception: + logging.exception('hhmm') + done.send_exception(*sys.exc_info()) + return + + done.send(True) + + greenthread.spawn(_inner) + return done + + def stop(self): + self._running = False + + diff --git a/nova/virt/fake.py b/nova/virt/fake.py index eaa2261f5c..0684a08779 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -24,8 +24,6 @@ This module also documents the semantics of real hypervisor connections. import logging -from twisted.internet import defer - from nova.compute import power_state @@ -105,7 +103,6 @@ class FakeConnection(object): fake_instance = FakeInstance() self.instances[instance.name] = fake_instance fake_instance._state = power_state.RUNNING - return defer.succeed(None) def reboot(self, instance): """ @@ -117,7 +114,7 @@ class FakeConnection(object): The work will be done asynchronously. This function returns a Deferred that allows the caller to detect when it is complete. """ - return defer.succeed(None) + pass def destroy(self, instance): """ @@ -130,7 +127,6 @@ class FakeConnection(object): Deferred that allows the caller to detect when it is complete. """ del self.instances[instance.name] - return defer.succeed(None) def attach_volume(self, instance_name, device_path, mountpoint): """Attach the disk at device_path to the instance at mountpoint""" diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 509ed97a05..9ca97bd1ba 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -25,10 +25,10 @@ import logging import os import shutil +from eventlet import event +from eventlet import tpool + import IPy -from twisted.internet import defer -from twisted.internet import task -from twisted.internet import threads from nova import context from nova import db @@ -145,13 +145,12 @@ class LibvirtConnection(object): except Exception as _err: pass # If the instance is already terminated, we're still happy - d = defer.Deferred() - d.addCallback(lambda _: self._cleanup(instance)) - # FIXME: What does this comment mean? - # TODO(termie): short-circuit me for tests - # WE'LL save this for when we do shutdown, + + done = event.Event() + + # We'll save this for when we do shutdown, # instead of destroy - but destroy returns immediately - timer = task.LoopingCall(f=None) + timer = utils.LoopingCall(f=None) def _wait_for_shutdown(): try: @@ -160,17 +159,26 @@ class LibvirtConnection(object): instance['id'], state) if state == power_state.SHUTDOWN: timer.stop() - d.callback(None) except Exception: db.instance_set_state(context.get_admin_context(), instance['id'], power_state.SHUTDOWN) timer.stop() - d.callback(None) timer.f = _wait_for_shutdown - timer.start(interval=0.5, now=True) - return d + timer_done = timer.start(interval=0.5, now=True) + + # NOTE(termie): this is strictly superfluous (we could put the + # cleanup code in the timer), but this emulates the + # previous model so I am keeping it around until + # everything has been vetted a bit + def _wait_for_timer(): + timer_done.wait() + self._cleanup(instance) + done.send() + + greenthread.spawn(_wait_for_time) + return done def _cleanup(self, instance): target = os.path.join(FLAGS.instances_path, instance['name']) @@ -179,32 +187,28 @@ class LibvirtConnection(object): if os.path.exists(target): shutil.rmtree(target) - @defer.inlineCallbacks @exception.wrap_exception def attach_volume(self, instance_name, device_path, mountpoint): - yield process.simple_execute("sudo virsh attach-disk %s %s %s" % - (instance_name, - device_path, - mountpoint.rpartition('/dev/')[2])) + process.simple_execute("sudo virsh attach-disk %s %s %s" % + (instance_name, + device_path, + mountpoint.rpartition('/dev/')[2])) - @defer.inlineCallbacks @exception.wrap_exception def detach_volume(self, instance_name, mountpoint): # NOTE(vish): despite the documentation, virsh detach-disk just # wants the device name without the leading /dev/ - yield process.simple_execute("sudo virsh detach-disk %s %s" % - (instance_name, - mountpoint.rpartition('/dev/')[2])) + process.simple_execute("sudo virsh detach-disk %s %s" % + (instance_name, + mountpoint.rpartition('/dev/')[2])) - @defer.inlineCallbacks @exception.wrap_exception def reboot(self, instance): xml = self.to_xml(instance) - yield self._conn.lookupByName(instance['name']).destroy() - yield self._conn.createXML(xml, 0) + self._conn.lookupByName(instance['name']).destroy() + self._conn.createXML(xml, 0) - d = defer.Deferred() - timer = task.LoopingCall(f=None) + timer = utils.LoopingCall(f=None) def _wait_for_reboot(): try: @@ -214,20 +218,16 @@ class LibvirtConnection(object): if state == power_state.RUNNING: logging.debug('instance %s: rebooted', instance['name']) timer.stop() - d.callback(None) except Exception, exn: logging.error('_wait_for_reboot failed: %s', exn) db.instance_set_state(context.get_admin_context(), instance['id'], power_state.SHUTDOWN) timer.stop() - d.callback(None) timer.f = _wait_for_reboot - timer.start(interval=0.5, now=True) - yield d + return timer.start(interval=0.5, now=True) - @defer.inlineCallbacks @exception.wrap_exception def spawn(self, instance): xml = self.to_xml(instance) @@ -235,16 +235,12 @@ class LibvirtConnection(object): instance['id'], power_state.NOSTATE, 'launching') - yield NWFilterFirewall(self._conn).\ - setup_nwfilters_for_instance(instance) - yield self._create_image(instance, xml) - yield self._conn.createXML(xml, 0) - # TODO(termie): this should actually register - # a callback to check for successful boot + NWFilterFirewall(self._conn).setup_nwfilters_for_instance(instance) + self._create_image(instance, xml) + self._conn.createXML(xml, 0) logging.debug("instance %s: is running", instance['name']) - local_d = defer.Deferred() - timer = task.LoopingCall(f=None) + timer = utils.LoopingCall(f=None) def _wait_for_boot(): try: @@ -254,7 +250,6 @@ class LibvirtConnection(object): if state == power_state.RUNNING: logging.debug('instance %s: booted', instance['name']) timer.stop() - local_d.callback(None) except: logging.exception('instance %s: failed to boot', instance['name']) @@ -262,10 +257,9 @@ class LibvirtConnection(object): instance['id'], power_state.SHUTDOWN) timer.stop() - local_d.callback(None) + timer.f = _wait_for_boot - timer.start(interval=0.5, now=True) - yield local_d + return timer.start(interval=0.5, now=True) def _flush_xen_console(self, virsh_output): logging.info('virsh said: %r' % (virsh_output,)) @@ -273,10 +267,9 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): logging.info('cool, it\'s a device') - d = process.simple_execute("sudo dd if=%s iflag=nonblock" % + r = process.simple_execute("sudo dd if=%s iflag=nonblock" % virsh_output, check_exit_code=False) - d.addCallback(lambda r: r[0]) - return d + return r[0] else: return '' @@ -296,21 +289,21 @@ class LibvirtConnection(object): def get_console_output(self, instance): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - d = process.simple_execute('sudo chown %d %s' % (os.getuid(), - console_log)) - if FLAGS.libvirt_type == 'xen': - # Xen is spethial - d.addCallback(lambda _: - process.simple_execute("virsh ttyconsole %s" % - instance['name'])) - d.addCallback(self._flush_xen_console) - d.addCallback(self._append_to_file, console_log) - else: - d.addCallback(lambda _: defer.succeed(console_log)) - d.addCallback(self._dump_file) - return d + + process.simple_execute('sudo chown %d %s' % (os.getuid(), + console_log)) + + if FLAGS.libvirt_type == 'xen': + # Xen is special + virsh_output = process.simple_execute("virsh ttyconsole %s" % + instance['name']) + data = self._flush_xen_console(virsh_output) + fpath = self._append_to_file(data, console_log) + else: + fpath = console_log + + return self._dump_file(fpath) - @defer.inlineCallbacks def _create_image(self, inst, libvirt_xml): # syntactic nicety basepath = lambda fname='': os.path.join(FLAGS.instances_path, @@ -318,8 +311,8 @@ class LibvirtConnection(object): fname) # ensure directories exist and are writable - yield process.simple_execute('mkdir -p %s' % basepath()) - yield process.simple_execute('chmod 0777 %s' % basepath()) + process.simple_execute('mkdir -p %s' % basepath()) + process.simple_execute('chmod 0777 %s' % basepath()) # TODO(termie): these are blocking calls, it would be great # if they weren't. @@ -335,19 +328,19 @@ class LibvirtConnection(object): project = manager.AuthManager().get_project(inst['project_id']) if not os.path.exists(basepath('disk')): - yield images.fetch(inst.image_id, basepath('disk-raw'), user, - project) + images.fetch(inst.image_id, basepath('disk-raw'), user, + project) if not os.path.exists(basepath('kernel')): - yield images.fetch(inst.kernel_id, basepath('kernel'), user, - project) + images.fetch(inst.kernel_id, basepath('kernel'), user, + project) if not os.path.exists(basepath('ramdisk')): - yield images.fetch(inst.ramdisk_id, basepath('ramdisk'), user, - project) - - execute = lambda cmd, process_input=None, check_exit_code=True: \ - process.simple_execute(cmd=cmd, - process_input=process_input, - check_exit_code=check_exit_code) + images.fetch(inst.ramdisk_id, basepath('ramdisk'), user, + project) + + def execute(cmd, process_input=None, check_exit_code=True): + return process.simple_execute(cmd=cmd, + process_input=process_input, + check_exit_code=check_exit_code) key = str(inst['key_data']) net = None @@ -369,23 +362,23 @@ class LibvirtConnection(object): if net: logging.info('instance %s: injecting net into image %s', inst['name'], inst.image_id) - yield disk.inject_data(basepath('disk-raw'), key, net, - execute=execute) + disk.inject_data(basepath('disk-raw'), key, net, + execute=execute) if os.path.exists(basepath('disk')): - yield process.simple_execute('rm -f %s' % basepath('disk')) + process.simple_execute('rm -f %s' % basepath('disk')) local_bytes = (instance_types.INSTANCE_TYPES[inst.instance_type] ['local_gb'] * 1024 * 1024 * 1024) resize = inst['instance_type'] != 'm1.tiny' - yield disk.partition(basepath('disk-raw'), basepath('disk'), - local_bytes, resize, execute=execute) + disk.partition(basepath('disk-raw'), basepath('disk'), + local_bytes, resize, execute=execute) if FLAGS.libvirt_type == 'uml': - yield process.simple_execute('sudo chown root %s' % - basepath('disk')) + process.simple_execute('sudo chown root %s' % + basepath('disk')) def to_xml(self, instance): # TODO(termie): cache? @@ -637,15 +630,15 @@ class NWFilterFirewall(object): def _define_filter(self, xml): if callable(xml): xml = xml() - d = threads.deferToThread(self._conn.nwfilterDefineXML, xml) - return d + + # execute in a native thread and block until done + tpool.execute(self._conn.nwfilterDefineXML, xml) @staticmethod def _get_net_and_mask(cidr): net = IPy.IP(cidr) return str(net.net()), str(net.netmask()) - @defer.inlineCallbacks def setup_nwfilters_for_instance(self, instance): """ Creates an NWFilter for the given instance. In the process, @@ -653,10 +646,10 @@ class NWFilterFirewall(object): the base filter are all in place. """ - yield self._define_filter(self.nova_base_ipv4_filter) - yield self._define_filter(self.nova_base_ipv6_filter) - yield self._define_filter(self.nova_dhcp_filter) - yield self._define_filter(self.nova_base_filter) + self._define_filter(self.nova_base_ipv4_filter) + self._define_filter(self.nova_base_ipv6_filter) + self._define_filter(self.nova_dhcp_filter) + self._define_filter(self.nova_base_filter) nwfilter_xml = "\n" \ " \n" % \ @@ -668,20 +661,19 @@ class NWFilterFirewall(object): net, mask = self._get_net_and_mask(network_ref['cidr']) project_filter = self.nova_project_filter(instance['project_id'], net, mask) - yield self._define_filter(project_filter) + self._define_filter(project_filter) nwfilter_xml += " \n" % \ instance['project_id'] for security_group in instance.security_groups: - yield self.ensure_security_group_filter(security_group['id']) + self.ensure_security_group_filter(security_group['id']) nwfilter_xml += " \n" % \ security_group['id'] nwfilter_xml += "" - yield self._define_filter(nwfilter_xml) - return + self._define_filter(nwfilter_xml) def ensure_security_group_filter(self, security_group_id): return self._define_filter( diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index a17e405ab1..f997d01d72 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -36,11 +36,10 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. """ import logging +import sys import xmlrpclib -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import task +from eventlet import tpool from nova import db from nova import flags @@ -110,36 +109,33 @@ class XenAPIConnection(object): return [self._conn.xenapi.VM.get_name_label(vm) \ for vm in self._conn.xenapi.VM.get_all()] - @defer.inlineCallbacks def spawn(self, instance): - vm = yield self._lookup(instance.name) + vm = self._lookup(instance.name) if vm is not None: raise Exception('Attempted to create non-unique name %s' % instance.name) network = db.project_get_network(None, instance.project_id) - network_ref = \ - yield self._find_network_with_bridge(network.bridge) + network_ref = self._find_network_with_bridge(network.bridge) user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - vdi_uuid = yield self._fetch_image( + vdi_uuid = self._fetch_image( instance.image_id, user, project, True) - kernel = yield self._fetch_image( + kernel = self._fetch_image( instance.kernel_id, user, project, False) - ramdisk = yield self._fetch_image( + ramdisk = self._fetch_image( instance.ramdisk_id, user, project, False) - vdi_ref = yield self._call_xenapi('VDI.get_by_uuid', vdi_uuid) + vdi_ref = self._call_xenapi('VDI.get_by_uuid', vdi_uuid) - vm_ref = yield self._create_vm(instance, kernel, ramdisk) - yield self._create_vbd(vm_ref, vdi_ref, 0, True) + vm_ref = self._create_vm(instance, kernel, ramdisk) + self._create_vbd(vm_ref, vdi_ref, 0, True) if network_ref: - yield self._create_vif(vm_ref, network_ref, instance.mac_address) + self._create_vif(vm_ref, network_ref, instance.mac_address) logging.debug('Starting VM %s...', vm_ref) - yield self._call_xenapi('VM.start', vm_ref, False, False) + self._call_xenapi('VM.start', vm_ref, False, False) logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - @defer.inlineCallbacks def _create_vm(self, instance, kernel, ramdisk): """Create a VM record. Returns a Deferred that gives the new VM reference.""" @@ -177,11 +173,10 @@ class XenAPIConnection(object): 'other_config': {}, } logging.debug('Created VM %s...', instance.name) - vm_ref = yield self._call_xenapi('VM.create', rec) + vm_ref = self._call_xenapi('VM.create', rec) logging.debug('Created VM %s as %s.', instance.name, vm_ref) - defer.returnValue(vm_ref) + return vm_ref - @defer.inlineCallbacks def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): """Create a VBD record. Returns a Deferred that gives the new VBD reference.""" @@ -200,12 +195,11 @@ class XenAPIConnection(object): vbd_rec['qos_algorithm_params'] = {} vbd_rec['qos_supported_algorithms'] = [] logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) - vbd_ref = yield self._call_xenapi('VBD.create', vbd_rec) + vbd_ref = self._call_xenapi('VBD.create', vbd_rec) logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, vdi_ref) - defer.returnValue(vbd_ref) + return vbd_ref - @defer.inlineCallbacks def _create_vif(self, vm_ref, network_ref, mac_address): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" @@ -221,24 +215,22 @@ class XenAPIConnection(object): vif_rec['qos_algorithm_params'] = {} logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, network_ref) - vif_ref = yield self._call_xenapi('VIF.create', vif_rec) + vif_ref = self._call_xenapi('VIF.create', vif_rec) logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, vm_ref, network_ref) - defer.returnValue(vif_ref) + return vif_ref - @defer.inlineCallbacks def _find_network_with_bridge(self, bridge): expr = 'field "bridge" = "%s"' % bridge - networks = yield self._call_xenapi('network.get_all_records_where', - expr) + networks = self._call_xenapi('network.get_all_records_where', + expr) if len(networks) == 1: - defer.returnValue(networks.keys()[0]) + return networks.keys()[0] elif len(networks) > 1: raise Exception('Found non-unique network for bridge %s' % bridge) else: raise Exception('Found no network for bridge %s' % bridge) - @defer.inlineCallbacks def _fetch_image(self, image, user, project, use_sr): """use_sr: True to put the image as a VDI in an SR, False to place it on dom0's filesystem. The former is for VM disks, the latter for @@ -255,33 +247,31 @@ class XenAPIConnection(object): args['password'] = user.secret if use_sr: args['add_partition'] = 'true' - task = yield self._async_call_plugin('objectstore', fn, args) - uuid = yield self._wait_for_task(task) - defer.returnValue(uuid) + task = self._async_call_plugin('objectstore', fn, args) + uuid = self._wait_for_task(task) + return uuid - @defer.inlineCallbacks def reboot(self, instance): - vm = yield self._lookup(instance.name) + vm = self._lookup(instance.name) if vm is None: raise Exception('instance not present %s' % instance.name) - task = yield self._call_xenapi('Async.VM.clean_reboot', vm) - yield self._wait_for_task(task) + task = self._call_xenapi('Async.VM.clean_reboot', vm) + self._wait_for_task(task) - @defer.inlineCallbacks def destroy(self, instance): - vm = yield self._lookup(instance.name) + vm = self._lookup(instance.name) if vm is None: # Don't complain, just return. This lets us clean up instances # that have already disappeared from the underlying platform. - defer.returnValue(None) + return try: - task = yield self._call_xenapi('Async.VM.hard_shutdown', vm) - yield self._wait_for_task(task) + task = self._call_xenapi('Async.VM.hard_shutdown', vm) + self._wait_for_task(task) except Exception, exc: logging.warn(exc) try: - task = yield self._call_xenapi('Async.VM.destroy', vm) - yield self._wait_for_task(task) + task = self._call_xenapi('Async.VM.destroy', vm) + self._wait_for_task(task) except Exception, exc: logging.warn(exc) @@ -299,7 +289,6 @@ class XenAPIConnection(object): def get_console_output(self, instance): return 'FAKE CONSOLE OUTPUT' - @utils.deferredToThread def _lookup(self, i): return self._lookup_blocking(i) @@ -316,35 +305,32 @@ class XenAPIConnection(object): def _wait_for_task(self, task): """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" - d = defer.Deferred() - reactor.callLater(0, self._poll_task, task, d) - return d - @utils.deferredToThread - def _poll_task(self, task, deferred): + done = event.Event() + loop = utis.LoopingTask(self._poll_task, task, done) + loop.start(FLAGS.xenapi_task_poll_interval, now=True) + return done.wait() + + def _poll_task(self, task, done): """Poll the given XenAPI task, and fire the given Deferred if we get a result.""" try: - #logging.debug('Polling task %s...', task) status = self._conn.xenapi.task.get_status(task) if status == 'pending': - reactor.callLater(FLAGS.xenapi_task_poll_interval, - self._poll_task, task, deferred) + return elif status == 'success': result = self._conn.xenapi.task.get_result(task) logging.info('Task %s status: success. %s', task, result) - deferred.callback(_parse_xmlrpc_value(result)) + done.send(_parse_xmlrpc_value(result)) else: error_info = self._conn.xenapi.task.get_error_info(task) logging.warn('Task %s status: %s. %s', task, status, error_info) - deferred.errback(XenAPI.Failure(error_info)) - #logging.debug('Polling task %s done.', task) + done.send_exception(XenAPI.Failure(error_info)) except Exception, exc: logging.warn(exc) - deferred.errback(exc) + done.send_exception(*sys.exc_info()) - @utils.deferredToThread def _call_xenapi(self, method, *args): """Call the specified XenAPI method on a background thread. Returns a Deferred for the result.""" @@ -353,11 +339,10 @@ class XenAPIConnection(object): f = f.__getattr__(m) return f(*args) - @utils.deferredToThread def _async_call_plugin(self, plugin, fn, args): """Call Async.host.call_plugin on a background thread. Returns a Deferred with the task reference.""" - return _unwrap_plugin_exceptions( + return tpool.execute(_unwrap_plugin_exceptions, self._conn.xenapi.Async.host.call_plugin, self._get_xenapi_host(), plugin, fn, args) From 33850cb57e195e538d6e42cb6d10f8296c0d4be4 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 25 Oct 2010 22:42:49 +0000 Subject: [PATCH 002/119] Moving the openldap schema out of nova.sh into it's own files, and adding sun (opends/opendj/sun directory server/fedora ds) schema files --- nova/auth/nova_openldap.schema | 84 +++++++++++++++++++ nova/auth/nova_sun.schema | 16 ++++ nova/auth/openssh-lpk_openldap.schema | 19 +++++ nova/auth/openssh-lpk_sun.schema | 3 + nova/auth/slap.sh | 112 +------------------------- 5 files changed, 125 insertions(+), 109 deletions(-) create mode 100644 nova/auth/nova_openldap.schema create mode 100644 nova/auth/nova_sun.schema create mode 100644 nova/auth/openssh-lpk_openldap.schema create mode 100644 nova/auth/openssh-lpk_sun.schema diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema new file mode 100644 index 0000000000..4047361de4 --- /dev/null +++ b/nova/auth/nova_openldap.schema @@ -0,0 +1,84 @@ +# +# Person object for Nova +# inetorgperson with extra attributes +# Author: Vishvananda Ishaya +# +# + +# using internet experimental oid arc as per BP64 3.1 +objectidentifier novaSchema 1.3.6.1.3.1.666.666 +objectidentifier novaAttrs novaSchema:3 +objectidentifier novaOCs novaSchema:4 + +attributetype ( + novaAttrs:1 + NAME 'accessKey' + DESC 'Key for accessing data' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:2 + NAME 'secretKey' + DESC 'Secret key' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:3 + NAME 'keyFingerprint' + DESC 'Fingerprint of private key' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:4 + NAME 'isAdmin' + DESC 'Is user an administrator?' + EQUALITY booleanMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:5 + NAME 'projectManager' + DESC 'Project Managers of a project' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + ) + +objectClass ( + novaOCs:1 + NAME 'novaUser' + DESC 'access and secret keys' + AUXILIARY + MUST ( uid ) + MAY ( accessKey $ secretKey $ isAdmin ) + ) + +objectClass ( + novaOCs:2 + NAME 'novaKeyPair' + DESC 'Key pair for User' + SUP top + STRUCTURAL + MUST ( cn $ sshPublicKey $ keyFingerprint ) + ) + +objectClass ( + novaOCs:3 + NAME 'novaProject' + DESC 'Container for project' + SUP groupOfNames + STRUCTURAL + MUST ( cn $ projectManager ) + ) diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema new file mode 100644 index 0000000000..e925e05e48 --- /dev/null +++ b/nova/auth/nova_sun.schema @@ -0,0 +1,16 @@ +# +# Person object for Nova +# inetorgperson with extra attributes +# Author: Vishvananda Ishaya +# Modified for strict RFC 4512 compatibility by: Ryan Lane +# +# using internet experimental oid arc as per BP64 3.1 +dn: cn=schema +attributeTypes: ( 1.3.6.1.3.1.666.666.3.1 NAME 'accessKey' DESC 'Key for accessing data' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.2 NAME 'secretKey' DESC 'Secret key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.3 NAME 'keyFingerprint' DESC 'Fingerprint of private key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isAdmin' DESC 'Is user an administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.5 NAME 'projectManager' DESC 'Project Managers of a project' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MUST ( uid ) MAY ( accessKey $ secretKey $ isAdmin ) ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.2 NAME 'novaKeyPair' DESC 'Key pair for User' SUP top STRUCTURAL MUST ( cn $ sshPublicKey $ keyFingerprint ) ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.3 NAME 'novaProject' DESC 'Container for project' SUP groupOfNames STRUCTURAL MUST ( cn $ projectManager ) ) diff --git a/nova/auth/openssh-lpk_openldap.schema b/nova/auth/openssh-lpk_openldap.schema new file mode 100644 index 0000000000..93351da6dd --- /dev/null +++ b/nova/auth/openssh-lpk_openldap.schema @@ -0,0 +1,19 @@ +# +# LDAP Public Key Patch schema for use with openssh-ldappubkey +# Author: Eric AUGE +# +# Based on the proposal of : Mark Ruijter +# + + +# octetString SYNTAX +attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' + DESC 'MANDATORY: OpenSSH Public key' + EQUALITY octetStringMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) + +# printableString SYNTAX yes|no +objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY + DESC 'MANDATORY: OpenSSH LPK objectclass' + MAY ( sshPublicKey $ uid ) + ) diff --git a/nova/auth/openssh-lpk_sun.schema b/nova/auth/openssh-lpk_sun.schema new file mode 100644 index 0000000000..5b220ab062 --- /dev/null +++ b/nova/auth/openssh-lpk_sun.schema @@ -0,0 +1,3 @@ +dn: cn=schema +attributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) +objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MAY ( sshPublicKey $ uid ) ) diff --git a/nova/auth/slap.sh b/nova/auth/slap.sh index fdc0e39dc1..797675d2e4 100755 --- a/nova/auth/slap.sh +++ b/nova/auth/slap.sh @@ -20,115 +20,9 @@ apt-get install -y slapd ldap-utils python-ldap -cat >/etc/ldap/schema/openssh-lpk_openldap.schema < -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) -LPK_SCHEMA_EOF - -cat >/etc/ldap/schema/nova.schema < -# -# - -# using internet experimental oid arc as per BP64 3.1 -objectidentifier novaSchema 1.3.6.1.3.1.666.666 -objectidentifier novaAttrs novaSchema:3 -objectidentifier novaOCs novaSchema:4 - -attributetype ( - novaAttrs:1 - NAME 'accessKey' - DESC 'Key for accessing data' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:2 - NAME 'secretKey' - DESC 'Secret key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - -objectClass ( - novaOCs:1 - NAME 'novaUser' - DESC 'access and secret keys' - AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) - -NOVA_SCHEMA_EOF +abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` +cp $abspath/openssh-lpk_openldap.schema /etc/ldap/schema/openssh-lpk_openldap.schema +cp $abspath/nova_openldap.schema /etc/ldap/schema/nova_openldap.schema mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig cat >/etc/ldap/slapd.conf < Date: Mon, 25 Oct 2010 22:50:32 +0000 Subject: [PATCH 003/119] Documentation was missing; added --- nova/auth/openssh-lpk_sun.schema | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/auth/openssh-lpk_sun.schema b/nova/auth/openssh-lpk_sun.schema index 5b220ab062..5f52db3b65 100644 --- a/nova/auth/openssh-lpk_sun.schema +++ b/nova/auth/openssh-lpk_sun.schema @@ -1,3 +1,10 @@ +# +# LDAP Public Key Patch schema for use with openssh-ldappubkey +# Author: Eric AUGE +# +# Schema for Sun Directory Server. +# Based on the original schema, modified by Stefan Fischer. +# dn: cn=schema attributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MAY ( sshPublicKey $ uid ) ) From a21c338c73c13281dfdd12ccb0c6168498855b9f Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Thu, 4 Nov 2010 15:50:23 -0700 Subject: [PATCH 004/119] Refactored smoketests to use novarc environment and to separate user and admin specific tests --- smoketests/admin_smoketests.py | 92 ++++ smoketests/{novatestcase.py => base.py} | 102 +++-- smoketests/flags.py | 13 +- smoketests/smoketest.py | 566 ------------------------ smoketests/user_smoketests.py | 326 ++++++++++++++ 5 files changed, 484 insertions(+), 615 deletions(-) create mode 100644 smoketests/admin_smoketests.py rename smoketests/{novatestcase.py => base.py} (51%) delete mode 100644 smoketests/smoketest.py create mode 100644 smoketests/user_smoketests.py diff --git a/smoketests/admin_smoketests.py b/smoketests/admin_smoketests.py new file mode 100644 index 0000000000..50bb3fa2e1 --- /dev/null +++ b/smoketests/admin_smoketests.py @@ -0,0 +1,92 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import random +import sys +import time +import unittest +import zipfile + +from nova import adminclient +from smoketests import flags +from smoketests import base + + +SUITE_NAMES = '[user]' + +FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) + +# TODO(devamcar): Use random tempfile +ZIP_FILENAME = '/tmp/nova-me-x509.zip' + +TEST_PREFIX = 'test%s' % int(random.random()*1000000) +TEST_USERNAME = '%suser' % TEST_PREFIX +TEST_PROJECTNAME = '%sproject' % TEST_PREFIX + + +class AdminSmokeTestCase(base.SmokeTestCase): + def setUp(self): + self.admin = adminclient.NovaAdminClient( + access_key=os.getenv('EC2_ACCESS_KEY'), + secret_key=os.getenv('EC2_SECRET_KEY'), + clc_url=os.getenv('EC2_URL'), + region=FLAGS.region) + + +class UserTests(AdminSmokeTestCase): + """ Test admin credentials and user creation. """ + + def test_001_admin_can_connect(self): + conn = self.admin.connection_for('admin', 'admin') + self.assert_(conn) + + def test_002_admin_can_create_user(self): + user = self.admin.create_user(TEST_USERNAME) + self.assertEqual(user.username, TEST_USERNAME) + + def test_003_admin_can_create_project(self): + project = self.admin.create_project(TEST_PROJECTNAME, + TEST_USERNAME) + self.assertEqual(project.projectname, TEST_PROJECTNAME) + + def test_004_user_can_download_credentials(self): + buf = self.admin.get_zip(TEST_USERNAME, TEST_PROJECTNAME) + output = open(ZIP_FILENAME, 'w') + output.write(buf) + output.close() + + zip = zipfile.ZipFile(ZIP_FILENAME, 'a', zipfile.ZIP_DEFLATED) + bad = zip.testzip() + zip.close() + + self.failIf(bad) + + def test_999_tearDown(self): + self.admin.delete_project(TEST_PROJECTNAME) + self.admin.delete_user(TEST_USERNAME) + try: + os.remove(ZIP_FILENAME) + except: + pass + +if __name__ == "__main__": + suites = {'user': unittest.makeSuite(UserTests)} + sys.exit(base.run_tests(suites)) + diff --git a/smoketests/novatestcase.py b/smoketests/base.py similarity index 51% rename from smoketests/novatestcase.py rename to smoketests/base.py index 513e0ca912..5a14d3e098 100644 --- a/smoketests/novatestcase.py +++ b/smoketests/base.py @@ -16,36 +16,26 @@ # License for the specific language governing permissions and limitations # under the License. +import boto import commands +import httplib import os +import paramiko import random import sys import unittest +from boto.ec2.regioninfo import RegionInfo - -import paramiko - -from nova import adminclient from smoketests import flags FLAGS = flags.FLAGS -class NovaTestCase(unittest.TestCase): - def setUp(self): - self.nova_admin = adminclient.NovaAdminClient( - access_key=FLAGS.admin_access_key, - secret_key=FLAGS.admin_secret_key, - clc_ip=FLAGS.clc_ip) - - def tearDown(self): - pass - +class SmokeTestCase(unittest.TestCase): def connect_ssh(self, ip, key_name): # TODO(devcamcar): set a more reasonable connection timeout time key = paramiko.RSAKey.from_private_key_file('/tmp/%s.pem' % key_name) client = paramiko.SSHClient() - client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip, username='root', pkey=key) stdin, stdout, stderr = client.exec_command('uptime') @@ -53,26 +43,50 @@ class NovaTestCase(unittest.TestCase): return client def can_ping(self, ip): - return commands.getstatusoutput('ping -c 1 %s' % ip)[0] == 0 + """ Attempt to ping the specified IP, and give up after 1 second. """ - @property - def admin(self): - return self.nova_admin.connection_for('admin') + # NOTE(devcamcar): ping timeout flag is different in OSX. + if sys.platform == 'darwin': + timeout_flag = 't' + else: + timeout_flag = 'w' - def connection_for(self, username): - return self.nova_admin.connection_for(username) + status, output = commands.getstatusoutput('ping -c1 -%s1 %s' % + (timeout_flag, ip)) + return status == 0 - def create_user(self, username): - return self.nova_admin.create_user(username) + def connection_for_env(self, **kwargs): + """ + Returns a boto ec2 connection for the current environment. + """ + access_key = os.getenv('EC2_ACCESS_KEY') + secret_key = os.getenv('EC2_SECRET_KEY') + clc_url = os.getenv('EC2_URL') - def get_user(self, username): - return self.nova_admin.get_user(username) + if not access_key or not secret_key or not clc_url: + raise Exception('Missing EC2 environment variables. Please source ' + 'the appropriate novarc file before running this ' + 'test.') - def delete_user(self, username): - return self.nova_admin.delete_user(username) + parts = self.split_clc_url(clc_url) + return boto.connect_ec2(aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + is_secure=parts['is_secure'], + region=RegionInfo(None, + 'nova', + parts['ip']), + port=parts['port'], + path='/services/Cloud', + **kwargs) - def get_signed_zip(self, username): - return self.nova_admin.get_zip(username) + def split_clc_url(self, clc_url): + """ + Splits a cloud controller endpoint url. + """ + parts = httplib.urlsplit(clc_url) + is_secure = parts.scheme == 'https' + ip, port = parts.netloc.split(':') + return {'ip': ip, 'port': int(port), 'is_secure': is_secure} def create_key_pair(self, conn, key_name): try: @@ -116,15 +130,25 @@ class NovaTestCase(unittest.TestCase): raise Exception(output) return True - def register_image(self, bucket_name, manifest): - conn = nova_admin.connection_for('admin') - return conn.register_image("%s/%s.manifest.xml" % (bucket_name, manifest)) +def run_tests(suites): + argv = FLAGS(sys.argv) - def setUp_test_image(self, image, kernel=False): - self.bundle_image(image, kernel=kernel) - bucket = "auto_test_%s" % int(random.random() * 1000000) - self.upload_image(bucket, image) - return self.register_image(bucket, image) + if not os.getenv('EC2_ACCESS_KEY'): + print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ + 'source the appropriate novarc file before ' \ + 'running this test.' + return 1 + + if FLAGS.suite: + try: + suite = suites[FLAGS.suite] + except KeyError: + print >> sys.stderr, 'Available test suites:', \ + ', '.join(suites.keys()) + return 1 + + unittest.TextTestRunner(verbosity=2).run(suite) + else: + for suite in suites.itervalues(): + unittest.TextTestRunner(verbosity=2).run(suite) - def tearDown_test_image(self, conn, image_id): - conn.deregister_image(image_id) diff --git a/smoketests/flags.py b/smoketests/flags.py index 3617fb797a..ae4d095085 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -1,7 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. +# Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -33,13 +33,6 @@ DEFINE_bool = DEFINE_bool # __GLOBAL FLAGS ONLY__ # Define any app-specific flags in their own files, docs at: # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 -DEFINE_string('admin_access_key', 'admin', 'Access key for admin user') -DEFINE_string('admin_secret_key', 'admin', 'Secret key for admin user') -DEFINE_string('clc_ip', '127.0.0.1', 'IP of cloud controller API') -DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', - 'Local kernel file to use for bundling tests') -DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', - 'Local image file to use for bundling tests') -#DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', -# 'AMI for cloudpipe vpn server') +DEFINE_string('region', 'nova', 'Region to use') +DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') diff --git a/smoketests/smoketest.py b/smoketests/smoketest.py deleted file mode 100644 index ad95114d40..0000000000 --- a/smoketests/smoketest.py +++ /dev/null @@ -1,566 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import commands -import os -import random -import re -import sys -import time -import unittest -import zipfile - - -import paramiko - -from smoketests import flags -from smoketests import novatestcase - -SUITE_NAMES = '[user, image, security, public_network, volume]' - -FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) - -# TODO(devamcar): Use random tempfile -ZIP_FILENAME = '/tmp/nova-me-x509.zip' - -data = {} - -test_prefix = 'test%s' % int(random.random()*1000000) -test_username = '%suser' % test_prefix -test_bucket = '%s_bucket' % test_prefix -test_key = '%s_key' % test_prefix - -# Test admin credentials and user creation -class UserTests(novatestcase.NovaTestCase): - def test_001_admin_can_connect(self): - conn = self.connection_for('admin') - self.assert_(conn) - - def test_002_admin_can_create_user(self): - userinfo = self.create_user(test_username) - self.assertEqual(userinfo.username, test_username) - - def test_003_user_can_download_credentials(self): - buf = self.get_signed_zip(test_username) - output = open(ZIP_FILENAME, 'w') - output.write(buf) - output.close() - - zip = zipfile.ZipFile(ZIP_FILENAME, 'a', zipfile.ZIP_DEFLATED) - bad = zip.testzip() - zip.close() - - self.failIf(bad) - - def test_999_tearDown(self): - self.delete_user(test_username) - user = self.get_user(test_username) - self.assert_(user is None) - try: - os.remove(ZIP_FILENAME) - except: - pass - -# Test image bundling, registration, and launching -class ImageTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user(test_username) - - def test_001_admin_can_bundle_image(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_image)) - - def test_002_admin_can_upload_image(self): - self.assertTrue(self.upload_image(test_bucket, FLAGS.bundle_image)) - - def test_003_admin_can_register_image(self): - image_id = self.register_image(test_bucket, FLAGS.bundle_image) - self.assert_(image_id is not None) - data['image_id'] = image_id - - def test_004_admin_can_bundle_kernel(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) - - def test_005_admin_can_upload_kernel(self): - self.assertTrue(self.upload_image(test_bucket, FLAGS.bundle_kernel)) - - def test_006_admin_can_register_kernel(self): - # FIXME(devcamcar): registration should verify that bucket/manifest - # exists before returning successfully. - kernel_id = self.register_image(test_bucket, FLAGS.bundle_kernel) - self.assert_(kernel_id is not None) - data['kernel_id'] = kernel_id - - def test_007_admin_images_are_available_within_10_seconds(self): - for i in xrange(10): - image = self.admin.get_image(data['image_id']) - if image and image.state == 'available': - break - time.sleep(1) - else: - print image.state - self.assert_(False) # wasn't available within 10 seconds - self.assert_(image.type == 'machine') - - for i in xrange(10): - kernel = self.admin.get_image(data['kernel_id']) - if kernel and kernel.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(kernel.type == 'kernel') - - def test_008_admin_can_describe_image_attribute(self): - attrs = self.admin.get_image_attribute(data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - - def test_009_me_cannot_see_non_public_images(self): - conn = self.connection_for(test_username) - images = conn.get_all_images(image_ids=[data['image_id']]) - self.assertEqual(len(images), 0) - - def test_010_admin_can_modify_image_launch_permission(self): - conn = self.connection_for(test_username) - - self.admin.modify_image_attribute(image_id=data['image_id'], - operation='add', - attribute='launchPermission', - groups='all') - - image = conn.get_image(data['image_id']) - self.assertEqual(image.id, data['image_id']) - - def test_011_me_can_list_public_images(self): - conn = self.connection_for(test_username) - images = conn.get_all_images(image_ids=[data['image_id']]) - self.assertEqual(len(images), 1) - pass - - def test_012_me_can_see_launch_permission(self): - attrs = self.admin.get_image_attribute(data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - self.assert_(attrs.groups[0], 'all') - - # FIXME: add tests that user can launch image - -# def test_013_user_can_launch_admin_public_image(self): -# # TODO: Use openwrt kernel instead of default kernel -# conn = self.connection_for(test_username) -# reservation = conn.run_instances(data['image_id']) -# self.assertEqual(len(reservation.instances), 1) -# data['my_instance_id'] = reservation.instances[0].id - -# def test_014_instances_launch_within_30_seconds(self): -# pass - -# def test_015_user_can_terminate(self): -# conn = self.connection_for(test_username) -# terminated = conn.terminate_instances( -# instance_ids=[data['my_instance_id']]) -# self.assertEqual(len(terminated), 1) - - def test_016_admin_can_deregister_kernel(self): - self.assertTrue(self.admin.deregister_image(data['kernel_id'])) - - def test_017_admin_can_deregister_image(self): - self.assertTrue(self.admin.deregister_image(data['image_id'])) - - def test_018_admin_can_delete_bundle(self): - self.assertTrue(self.delete_bundle_bucket(test_bucket)) - - def test_999_tearDown(self): - data = {} - self.delete_user(test_username) - - -# Test key pairs and security groups -class SecurityTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user(test_username + '_me') - self.create_user(test_username + '_you') - data['image_id'] = 'ami-tiny' - - def test_001_me_can_create_keypair(self): - conn = self.connection_for(test_username + '_me') - key = self.create_key_pair(conn, test_key) - self.assertEqual(key.name, test_key) - - def test_002_you_can_create_keypair(self): - conn = self.connection_for(test_username + '_you') - key = self.create_key_pair(conn, test_key+ 'yourkey') - self.assertEqual(key.name, test_key+'yourkey') - - def test_003_me_can_create_instance_with_keypair(self): - conn = self.connection_for(test_username + '_me') - reservation = conn.run_instances(data['image_id'], key_name=test_key) - self.assertEqual(len(reservation.instances), 1) - data['my_instance_id'] = reservation.instances[0].id - - def test_004_me_can_obtain_private_ip_within_60_seconds(self): - conn = self.connection_for(test_username + '_me') - reservations = conn.get_all_instances([data['my_instance_id']]) - instance = reservations[0].instances[0] - # allow 60 seconds to exit pending with IP - for x in xrange(60): - instance.update() - if instance.state != u'pending': - break - time.sleep(1) - else: - self.assert_(False) - # self.assertEqual(instance.state, u'running') - ip = reservations[0].instances[0].private_dns_name - self.failIf(ip == '0.0.0.0') - data['my_private_ip'] = ip - print data['my_private_ip'], - - def test_005_can_ping_private_ip(self): - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 -w1 %s' % data['my_private_ip']) - if status == 0: - break - else: - self.assert_('could not ping instance') - #def test_005_me_cannot_ssh_when_unauthorized(self): - # self.assertRaises(paramiko.SSHException, self.connect_ssh, - # data['my_private_ip'], 'mykey') - - #def test_006_me_can_authorize_ssh(self): - # conn = self.connection_for(test_username + '_me') - # self.assertTrue( - # conn.authorize_security_group( - # 'default', - # ip_protocol='tcp', - # from_port=22, - # to_port=22, - # cidr_ip='0.0.0.0/0' - # ) - # ) - - def test_007_me_can_ssh_when_authorized(self): - conn = self.connect_ssh(data['my_private_ip'], test_key) - conn.close() - - #def test_008_me_can_revoke_ssh_authorization(self): - # conn = self.connection_for('me') - # self.assertTrue( - # conn.revoke_security_group( - # 'default', - # ip_protocol='tcp', - # from_port=22, - # to_port=22, - # cidr_ip='0.0.0.0/0' - # ) - # ) - - #def test_009_you_cannot_ping_my_instance(self): - # TODO: should ping my_private_ip from with an instance started by you. - #self.assertFalse(self.can_ping(data['my_private_ip'])) - - def test_010_you_cannot_ssh_to_my_instance(self): - try: - conn = self.connect_ssh(data['my_private_ip'], - test_key + 'yourkey') - conn.close() - except paramiko.SSHException: - pass - else: - self.fail("expected SSHException") - - def test_999_tearDown(self): - conn = self.connection_for(test_username + '_me') - self.delete_key_pair(conn, test_key) - if data.has_key('my_instance_id'): - conn.terminate_instances([data['my_instance_id']]) - - conn = self.connection_for(test_username + '_you') - self.delete_key_pair(conn, test_key + 'yourkey') - - conn = self.connection_for('admin') - self.delete_user(test_username + '_me') - self.delete_user(test_username + '_you') - #self.tearDown_test_image(conn, data['image_id']) - -# TODO: verify wrt image boots -# build python into wrt image -# build boto/m2crypto into wrt image -# build euca2ools into wrt image -# build a script to download and unpack credentials -# - return "ok" to stdout for comparison in self.assertEqual() -# build a script to bundle the instance -# build a script to upload the bundle - -# status, output = commands.getstatusoutput('cmd') -# if status == 0: -# print 'ok' -# else: -# print output - -# Testing rebundling -class RebundlingTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user('me') - self.create_user('you') - # TODO: create keypair for me - # upload smoketest img - # run instance - - def test_001_me_can_download_credentials_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/install-credentials.py') - conn.close() - self.assertEqual(stdout, 'ok') - - def test_002_me_can_rebundle_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/rebundle-instance.py') - conn.close() - self.assertEqual(stdout, 'ok') - - def test_003_me_can_upload_image_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/upload-bundle.py') - conn.close() - self.assertEqual(stdout, 'ok') - - def test_004_me_can_register_image_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/register-image.py') - conn.close() - if re.matches('ami-{\w+}', stdout): - data['my_image_id'] = stdout.strip() - else: - self.fail('expected ami-nnnnnn, got:\n ' + stdout) - - def test_005_you_cannot_see_my_private_image(self): - conn = self.connection_for('you') - image = conn.get_image(data['my_image_id']) - self.assertEqual(image, None) - - def test_006_me_can_make_image_public(self): - conn = self.connection_for(test_username) - conn.modify_image_attribute(image_id=data['my_image_id'], - operation='add', - attribute='launchPermission', - groups='all') - - def test_007_you_can_see_my_public_image(self): - conn = self.connection_for('you') - image = conn.get_image(data['my_image_id']) - self.assertEqual(image.id, data['my_image_id']) - - def test_999_tearDown(self): - self.delete_user('me') - self.delete_user('you') - - #if data.has_key('image_id'): - # deregister rebundled image - - # TODO: tear down instance - # delete keypairs - data = {} - -# Test elastic IPs -class ElasticIPTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - data['image_id'] = 'ami-tiny' - - self.create_user('me') - conn = self.connection_for('me') - self.create_key_pair(conn, 'mykey') - - conn = self.connection_for('admin') - #data['image_id'] = self.setUp_test_image(FLAGS.bundle_image) - - def test_001_me_can_launch_image_with_keypair(self): - conn = self.connection_for('me') - reservation = conn.run_instances(data['image_id'], key_name='mykey') - self.assertEqual(len(reservation.instances), 1) - data['my_instance_id'] = reservation.instances[0].id - - def test_002_me_can_allocate_elastic_ip(self): - conn = self.connection_for('me') - data['my_public_ip'] = conn.allocate_address() - self.assert_(data['my_public_ip'].public_ip) - - def test_003_me_can_associate_ip_with_instance(self): - self.assertTrue(data['my_public_ip'].associate(data['my_instance_id'])) - - def test_004_me_can_ssh_with_public_ip(self): - conn = self.connect_ssh(data['my_public_ip'].public_ip, 'mykey') - conn.close() - - def test_005_me_can_disassociate_ip_from_instance(self): - self.assertTrue(data['my_public_ip'].disassociate()) - - def test_006_me_can_deallocate_elastic_ip(self): - self.assertTrue(data['my_public_ip'].delete()) - - def test_999_tearDown(self): - conn = self.connection_for('me') - self.delete_key_pair(conn, 'mykey') - - conn = self.connection_for('admin') - #self.tearDown_test_image(conn, data['image_id']) - data = {} - -ZONE = 'nova' -DEVICE = 'vdb' -# Test iscsi volumes -class VolumeTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user(test_username) - data['image_id'] = 'ami-tiny' # A7370FE3 - - conn = self.connection_for(test_username) - self.create_key_pair(conn, test_key) - reservation = conn.run_instances(data['image_id'], - instance_type='m1.tiny', - key_name=test_key) - data['instance_id'] = reservation.instances[0].id - data['private_ip'] = reservation.instances[0].private_dns_name - # wait for instance to show up - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 -w1 %s' % data['private_ip']) - if status == 0: - break - else: - self.fail('unable to ping instance') - - def test_001_me_can_create_volume(self): - conn = self.connection_for(test_username) - volume = conn.create_volume(1, ZONE) - self.assertEqual(volume.size, 1) - data['volume_id'] = volume.id - # give network time to find volume - time.sleep(5) - - def test_002_me_can_attach_volume(self): - conn = self.connection_for(test_username) - conn.attach_volume( - volume_id = data['volume_id'], - instance_id = data['instance_id'], - device = '/dev/%s' % DEVICE - ) - # give instance time to recognize volume - time.sleep(5) - - def test_003_me_can_mount_volume(self): - conn = self.connect_ssh(data['private_ip'], test_key) - # FIXME(devcamcar): the tiny image doesn't create the node properly - # this will make /dev/vd* if it doesn't exist - stdin, stdout, stderr = conn.exec_command( - 'grep %s /proc/partitions |' + \ - '`awk \'{print "mknod /dev/"$4" b "$1" "$2}\'`' % DEVICE) - commands = [] - commands.append('mkdir -p /mnt/vol') - commands.append('mkfs.ext2 /dev/%s' % DEVICE) - commands.append('mount /dev/%s /mnt/vol' % DEVICE) - commands.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) - out = stdout.read() - conn.close() - if not out.strip().endswith('success'): - self.fail('Unable to mount: %s %s' % (out, stderr.read())) - - def test_004_me_can_write_to_volume(self): - conn = self.connect_ssh(data['private_ip'], test_key) - # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted - stdin, stdout, stderr = conn.exec_command( - 'echo hello > /mnt/vol/test.txt') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to write to mount: %s' % (err)) - - def test_005_volume_is_correct_size(self): - conn = self.connect_ssh(data['private_ip'], test_key) - stdin, stdout, stderr = conn.exec_command( - "df -h | grep %s | awk {'print $2'}" % DEVICE) - out = stdout.read() - conn.close() - if not out.strip() == '1007.9M': - self.fail('Volume is not the right size: %s %s' % (out, stderr.read())) - - def test_006_me_can_umount_volume(self): - conn = self.connect_ssh(data['private_ip'], test_key) - stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to unmount: %s' % (err)) - - def test_007_me_can_detach_volume(self): - conn = self.connection_for(test_username) - self.assertTrue(conn.detach_volume(volume_id = data['volume_id'])) - - def test_008_me_can_delete_volume(self): - conn = self.connection_for(test_username) - self.assertTrue(conn.delete_volume(data['volume_id'])) - - def test_009_volume_size_must_be_int(self): - conn = self.connection_for(test_username) - self.assertRaises(Exception, conn.create_volume, 'foo', ZONE) - - def test_999_tearDown(self): - global data - conn = self.connection_for(test_username) - self.delete_key_pair(conn, test_key) - if data.has_key('instance_id'): - conn.terminate_instances([data['instance_id']]) - self.delete_user(test_username) - data = {} - -def build_suites(): - return { - 'user': unittest.makeSuite(UserTests), - 'image': unittest.makeSuite(ImageTests), - 'security': unittest.makeSuite(SecurityTests), - 'public_network': unittest.makeSuite(ElasticIPTests), - 'volume': unittest.makeSuite(VolumeTests), - } - -def main(): - argv = FLAGS(sys.argv) - suites = build_suites() - - if FLAGS.suite: - try: - suite = suites[FLAGS.suite] - except KeyError: - print >> sys.stderr, 'Available test suites:', SUITE_NAMES - return 1 - - unittest.TextTestRunner(verbosity=2).run(suite) - else: - for suite in suites.itervalues(): - unittest.TextTestRunner(verbosity=2).run(suite) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py new file mode 100644 index 0000000000..d29e3aea32 --- /dev/null +++ b/smoketests/user_smoketests.py @@ -0,0 +1,326 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import socket +import sys +import time +import unittest + +from smoketests import flags +from smoketests import base + + +SUITE_NAMES = '[image, instance, volume]' + +FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) +flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', + 'Local kernel file to use for bundling tests') +flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', + 'Local image file to use for bundling tests') + +TEST_PREFIX = 'test%s' % int (random.random()*1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_DATA = {} + + +class UserSmokeTestCase(base.SmokeTestCase): + def setUp(self): + global TEST_DATA + self.conn = self.connection_for_env() + self.data = TEST_DATA + + +class ImageTests(UserSmokeTestCase): + def test_001_can_bundle_image(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_image)) + + def test_002_can_upload_image(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) + + def test_003_can_register_image(self): + image_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_image)) + self.assert_(image_id is not None) + self.data['image_id'] = image_id + + def test_004_can_bundle_kernel(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) + + def test_005_can_upload_kernel(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) + + def test_006_can_register_kernel(self): + kernel_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_kernel)) + self.assert_(kernel_id is not None) + self.data['kernel_id'] = kernel_id + + def test_007_images_are_available_within_10_seconds(self): + for i in xrange(10): + image = self.conn.get_image(self.data['image_id']) + if image and image.state == 'available': + break + time.sleep(1) + else: + print image.state + self.assert_(False) # wasn't available within 10 seconds + self.assert_(image.type == 'machine') + + for i in xrange(10): + kernel = self.conn.get_image(self.data['kernel_id']) + if kernel and kernel.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(kernel.type == 'kernel') + + def test_008_can_describe_image_attribute(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + + def test_009_can_modify_image_launch_permission(self): + self.conn.modify_image_attribute(image_id=self.data['image_id'], + operation='add', + attribute='launchPermission', + groups='all') + image = self.conn.get_image(self.data['image_id']) + self.assertEqual(image.id, self.data['image_id']) + + def test_010_can_see_launch_permission(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + self.assert_(attrs.attrs['groups'][0], 'all') + + def test_011_user_can_deregister_kernel(self): + self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) + + def test_012_can_deregister_image(self): + self.assertTrue(self.conn.deregister_image(self.data['image_id'])) + + def test_013_can_delete_bundle(self): + self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) + + +class InstanceTests(UserSmokeTestCase): + def test_001_can_create_keypair(self): + key = self.create_key_pair(self.conn, TEST_KEY) + self.assertEqual(key.name, TEST_KEY) + + def test_002_can_create_instance_with_keypair(self): + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + instance_type='m1.tiny') + self.assertEqual(len(reservation.instances), 1) + self.data['instance_id'] = reservation.instances[0].id + + def test_003_instance_runs_within_60_seconds(self): + reservations = self.conn.get_all_instances([data['instance_id']]) + instance = reservations[0].instances[0] + # allow 60 seconds to exit pending with IP + for x in xrange(60): + instance.update() + if instance.state == u'running': + break + time.sleep(1) + else: + self.fail('instance failed to start') + ip = reservations[0].instances[0].private_dns_name + self.failIf(ip == '0.0.0.0') + self.data['private_ip'] = ip + print self.data['private_ip'] + + def test_004_can_ping_private_ip(self): + for x in xrange(120): + # ping waits for 1 second + status, output = commands.getstatusoutput( + 'ping -c1 %s' % self.data['private_ip']) + if status == 0: + break + else: + self.fail('could not ping instance') + + def test_005_can_ssh_to_private_ip(self): + for x in xrange(30): + try: + conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) + conn.close() + except Exception: + time.sleep(1) + else: + break + else: + self.fail('could not ssh to instance') + + def test_006_can_allocate_elastic_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + def test_007_can_associate_ip_with_instance(self): + result = self.conn.associate_address(self.data['instance_id'], + self.data['public_ip']) + self.assertTrue(result) + + def test_008_can_ssh_with_public_ip(self): + for x in xrange(30): + try: + conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) + conn.close() + except socket.error: + time.sleep(1) + else: + break + else: + self.fail('could not ssh to instance') + + def test_009_can_disassociate_ip_from_instance(self): + result = self.conn.disassociate_address(self.data['public_ip']) + self.assertTrue(result) + + def test_010_can_deallocate_elastic_ip(self): + result = self.conn.release_address(self.data['public_ip']) + self.assertTrue(result) + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + if self.data.has_key('instance_id'): + self.conn.terminate_instances([data['instance_id']]) + + +class VolumeTests(UserSmokeTestCase): + def setUp(self): + super(VolumeTests, self).setUp() + self.device = '/dev/vdb' + + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + instance = reservation.instances[0] + self.data['instance'] = instance + for x in xrange(120): + if self.can_ping(instance.private_dns_name): + break + else: + self.fail('unable to start instance') + + def test_001_can_create_volume(self): + volume = self.conn.create_volume(1, 'nova') + self.assertEqual(volume.size, 1) + self.data['volume'] = volume + # Give network time to find volume. + time.sleep(5) + + def test_002_can_attach_volume(self): + volume = self.data['volume'] + + for x in xrange(10): + if volume.status == u'available': + break + time.sleep(5) + volume.update() + else: + self.fail('cannot attach volume with state %s' % volume.status) + + volume.attach(self.data['instance'].id, self.device) + + # Volumes seems to report "available" too soon. + for x in xrange(10): + if volume.status == u'in-use': + break + time.sleep(5) + volume.update() + + self.assertEqual(volume.status, u'in-use') + + # Give instance time to recognize volume. + time.sleep(5) + + def test_003_can_mount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + commands = [] + commands.append('mkdir -p /mnt/vol') + commands.append('mkfs.ext2 %s' % self.device) + commands.append('mount %s /mnt/vol' % self.device) + commands.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) + out = stdout.read() + conn.close() + if not out.strip().endswith('success'): + self.fail('Unable to mount: %s %s' % (out, stderr.read())) + + def test_004_can_write_to_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted + stdin, stdout, stderr = conn.exec_command( + 'echo hello > /mnt/vol/test.txt') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to write to mount: %s' % (err)) + + def test_005_volume_is_correct_size(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command( + "df -h | grep %s | awk {'print $2'}" % self.device) + out = stdout.read() + conn.close() + if not out.strip() == '1008M': + self.fail('Volume is not the right size: %s %s' % + (out, stderr.read())) + + def test_006_me_can_umount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to unmount: %s' % (err)) + + def test_007_me_can_detach_volume(self): + result = self.conn.detach_volume(volume_id=self.data['volume'].id) + self.assertTrue(result) + time.sleep(5) + + def test_008_me_can_delete_volume(self): + result = self.conn.delete_volume(self.data['volume'].id) + self.assertTrue(result) + + def test_999_tearDown(self): + self.conn.terminate_instances([self.data['instance'].id]) + self.conn.delete_key_pair(TEST_KEY) + + +if __name__ == "__main__": + suites = {'image': unittest.makeSuite(ImageTests), + 'instance': unittest.makeSuite(InstanceTests), + 'volume': unittest.makeSuite(VolumeTests)} + sys.exit(base.run_tests(suites)) From 398ec325160acf039e1070718e898339a4b2d268 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 15 Nov 2010 01:25:42 -0400 Subject: [PATCH 005/119] base commit --- nova/api/openstack/__init__.py | 47 +++++++++---- .../tests/api/openstack/test_restrictedapi.py | 70 +++++++++++++++++++ 2 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 nova/tests/api/openstack/test_restrictedapi.py diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 1dd3ba770e..338d642bca 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -47,6 +47,9 @@ flags.DEFINE_string('nova_api_auth', 'nova.api.openstack.auth.BasicApiAuthManager', 'The auth mechanism to use for the OpenStack API implemenation') +flags.DEFINE_list('nova_api_permitted_operations', + [], + 'A comma-separated list of permitted api operations. Empty for all.') class API(wsgi.Middleware): """WSGI entry point for all OpenStack API requests.""" @@ -165,21 +168,37 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - mapper.resource("server", "servers", controller=servers.Controller(), - collection={'detail': 'GET'}, - member={'action': 'POST'}) + commands = { + "server" : dict(plural='servers', + controller=servers.Controller(), + collection={'detail': 'GET'}, + member={'action': 'POST'}), + "backup_schedule" : dict(plural='backup_schedules', + controller=backup_schedules.Controller(), + parent_resource=dict(member_name='server', + collection_name='servers')), + "image" : dict(plural='images', + controller=images.Controller(), + collection={'detail': 'GET'}), + "flavor" : dict(plural='flavors', + controller=flavors.Controller(), + collection={'detail': 'GET'}), + "sharedipgroup" : dict(plural="sharedipgroups", + controller=sharedipgroups.Controller()), + } - mapper.resource("backup_schedule", "backup_schedules", - controller=backup_schedules.Controller(), - parent_resource=dict(member_name='server', - collection_name='servers')) - - mapper.resource("image", "images", controller=images.Controller(), - collection={'detail': 'GET'}) - mapper.resource("flavor", "flavors", controller=flavors.Controller(), - collection={'detail': 'GET'}) - mapper.resource("sharedipgroup", "sharedipgroups", - controller=sharedipgroups.Controller()) + permitted = commands.keys() + if len(FLAGS.nova_api_permitted_operations) > 0: + permitted = FLAGS.nova_api_permitted_operations + logging.debug("Permitted operation set: %s" % (permitted,)) + for command in permitted: + options = commands.get(command, None) + if not options: + logging.warning("Unknown option in nova_api_permitted_operations: '%s' (skipping)" % (command,)) + continue + collection = options['plural'] + del options['plural'] + mapper.resource(command, collection, **options) super(APIRouter, self).__init__(mapper) diff --git a/nova/tests/api/openstack/test_restrictedapi.py b/nova/tests/api/openstack/test_restrictedapi.py new file mode 100644 index 0000000000..560a66e8c8 --- /dev/null +++ b/nova/tests/api/openstack/test_restrictedapi.py @@ -0,0 +1,70 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +import stubout +import webob + +import nova.api +from nova import flags +from nova.api.openstack import flavors +from nova.tests.api.openstack import fakes + +FLAGS = flags.FLAGS + +class RestrictedAPITest(unittest.TestCase): + def setUp(self): + self.stubs = stubout.StubOutForTesting() + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + self.original_permissions = FLAGS.nova_api_permitted_operations + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.nova_api_permitted_operations = self.original_permissions + + def test_permitted(self): + req = webob.Request.blank('/v1.0/flavors') + FLAGS.nova_api_permitted_operations = ["server", "backup_schedule", "image", "flavor", "sharedipgroup"] + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 200) + + def test_bad_list(self): + req = webob.Request.blank('/v1.0/flavors') + FLAGS.nova_api_permitted_operations = ["foo", "bar", "zoo"] + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 404) + + def test_default_all_permitted(self): + req = webob.Request.blank('/v1.0/flavors') + # empty means all operations available. + FLAGS.nova_api_permitted_operations = [] + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 200) + + def test_disallowed(self): + req = webob.Request.blank('/v1.0/flavors') + FLAGS.nova_api_permitted_operations = ["server", "backup_schedule", "image", "sharedipgroup"] + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 404) + +if __name__ == '__main__': + unittest.main() From 66a5ac31c4a5f24da9c0335cf934bbf545c0d95f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 16 Nov 2010 02:54:13 -0400 Subject: [PATCH 006/119] Changed from fine-grained operation control to binary admin on/off setting. --- nova/api/openstack/__init__.py | 54 ++++++++----------- ...test_restrictedapi.py => test_adminapi.py} | 30 ++++------- 2 files changed, 31 insertions(+), 53 deletions(-) rename nova/tests/api/openstack/{test_restrictedapi.py => test_adminapi.py} (61%) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 338d642bca..23ac033cf3 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -47,9 +47,9 @@ flags.DEFINE_string('nova_api_auth', 'nova.api.openstack.auth.BasicApiAuthManager', 'The auth mechanism to use for the OpenStack API implemenation') -flags.DEFINE_list('nova_api_permitted_operations', - [], - 'A comma-separated list of permitted api operations. Empty for all.') +flags.DEFINE_bool('allow_admin_api', + False, + 'When True, this API service will accept admin operations.') class API(wsgi.Middleware): """WSGI entry point for all OpenStack API requests.""" @@ -168,37 +168,25 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - commands = { - "server" : dict(plural='servers', - controller=servers.Controller(), - collection={'detail': 'GET'}, - member={'action': 'POST'}), - "backup_schedule" : dict(plural='backup_schedules', - controller=backup_schedules.Controller(), - parent_resource=dict(member_name='server', - collection_name='servers')), - "image" : dict(plural='images', - controller=images.Controller(), - collection={'detail': 'GET'}), - "flavor" : dict(plural='flavors', - controller=flavors.Controller(), - collection={'detail': 'GET'}), - "sharedipgroup" : dict(plural="sharedipgroups", - controller=sharedipgroups.Controller()), - } + mapper.resource("server", "servers", controller=servers.Controller(), + collection={'detail': 'GET'}, + member={'action': 'POST'}) - permitted = commands.keys() - if len(FLAGS.nova_api_permitted_operations) > 0: - permitted = FLAGS.nova_api_permitted_operations - logging.debug("Permitted operation set: %s" % (permitted,)) - for command in permitted: - options = commands.get(command, None) - if not options: - logging.warning("Unknown option in nova_api_permitted_operations: '%s' (skipping)" % (command,)) - continue - collection = options['plural'] - del options['plural'] - mapper.resource(command, collection, **options) + mapper.resource("backup_schedule", "backup_schedules", + controller=backup_schedules.Controller(), + parent_resource=dict(member_name='server', + collection_name='servers')) + + mapper.resource("image", "images", controller=images.Controller(), + collection={'detail': 'GET'}) + mapper.resource("flavor", "flavors", controller=flavors.Controller(), + collection={'detail': 'GET'}) + mapper.resource("sharedipgroup", "sharedipgroups", + controller=sharedipgroups.Controller()) + + if FLAGS.allow_admin_api: + logging.debug("Including admin operations in API.") + # TODO: Place routes for admin operations here. super(APIRouter, self).__init__(mapper) diff --git a/nova/tests/api/openstack/test_restrictedapi.py b/nova/tests/api/openstack/test_adminapi.py similarity index 61% rename from nova/tests/api/openstack/test_restrictedapi.py rename to nova/tests/api/openstack/test_adminapi.py index 560a66e8c8..54cb40e842 100644 --- a/nova/tests/api/openstack/test_restrictedapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -22,7 +22,6 @@ import webob import nova.api from nova import flags -from nova.api.openstack import flavors from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS @@ -35,36 +34,27 @@ class RestrictedAPITest(unittest.TestCase): fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) fakes.stub_out_auth(self.stubs) - self.original_permissions = FLAGS.nova_api_permitted_operations + self.allow_admin = FLAGS.allow_admin_api def tearDown(self): self.stubs.UnsetAll() - FLAGS.nova_api_permitted_operations = self.original_permissions + FLAGS.allow_admin_api = self.allow_admin - def test_permitted(self): + def test_admin_enabled(self): + FLAGS.allow_admin_api = True + # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') - FLAGS.nova_api_permitted_operations = ["server", "backup_schedule", "image", "flavor", "sharedipgroup"] res = req.get_response(nova.api.API('os')) self.assertEqual(res.status_int, 200) + # TODO: Confirm admin operations are available. - def test_bad_list(self): + def test_admin_disabled(self): + FLAGS.allow_admin_api = False + # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') - FLAGS.nova_api_permitted_operations = ["foo", "bar", "zoo"] - res = req.get_response(nova.api.API('os')) - self.assertEqual(res.status_int, 404) - - def test_default_all_permitted(self): - req = webob.Request.blank('/v1.0/flavors') - # empty means all operations available. - FLAGS.nova_api_permitted_operations = [] res = req.get_response(nova.api.API('os')) self.assertEqual(res.status_int, 200) - - def test_disallowed(self): - req = webob.Request.blank('/v1.0/flavors') - FLAGS.nova_api_permitted_operations = ["server", "backup_schedule", "image", "sharedipgroup"] - res = req.get_response(nova.api.API('os')) - self.assertEqual(res.status_int, 404) + # TODO: Confirm admin operations are unavailable. if __name__ == '__main__': unittest.main() From 10756392157aa5e6029a50a9f38718f3024731c7 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 16 Nov 2010 05:46:40 -0400 Subject: [PATCH 007/119] added myself to Authors file. Enjoy spiders. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index ef1a535ca9..22acc34d23 100644 --- a/Authors +++ b/Authors @@ -20,6 +20,7 @@ Michael Gundlach Monty Taylor Paul Voccio Rick Clark +Sandy Walsh Soren Hansen Todd Willey Vishvananda Ishaya From f0b53131569cd409a95c68b435ec56a69dcdc897 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 16 Nov 2010 05:53:21 -0400 Subject: [PATCH 008/119] PEP8 fixes --- nova/api/openstack/__init__.py | 1 + nova/tests/api/openstack/test_adminapi.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 23ac033cf3..80b27c7e55 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -51,6 +51,7 @@ flags.DEFINE_bool('allow_admin_api', False, 'When True, this API service will accept admin operations.') + class API(wsgi.Middleware): """WSGI entry point for all OpenStack API requests.""" diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 54cb40e842..1b2e1654d5 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -26,7 +26,8 @@ from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS -class RestrictedAPITest(unittest.TestCase): + +class AdminAPITest(unittest.TestCase): def setUp(self): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} @@ -42,7 +43,7 @@ class RestrictedAPITest(unittest.TestCase): def test_admin_enabled(self): FLAGS.allow_admin_api = True - # We should still be able to access public operations. + # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') res = req.get_response(nova.api.API('os')) self.assertEqual(res.status_int, 200) @@ -50,7 +51,7 @@ class RestrictedAPITest(unittest.TestCase): def test_admin_disabled(self): FLAGS.allow_admin_api = False - # We should still be able to access public operations. + # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') res = req.get_response(nova.api.API('os')) self.assertEqual(res.status_int, 200) From 0b70b44c11830549938c5153b4322b960c53963d Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 22 Nov 2010 16:43:17 -0600 Subject: [PATCH 009/119] Incorporating more networking info --- doc/source/adminguide/index.rst | 6 +++--- doc/source/adminguide/managing.networks.rst | 15 --------------- doc/source/adminguide/network.vlan.rst | 8 ++++---- doc/source/nova.concepts.rst | 21 +++++++++++++++++---- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/doc/source/adminguide/index.rst b/doc/source/adminguide/index.rst index 51228b3191..736a154b21 100644 --- a/doc/source/adminguide/index.rst +++ b/doc/source/adminguide/index.rst @@ -38,14 +38,14 @@ There are two main tools that a system administrator will find useful to manage nova.manage euca2ools -nova-manage may only be run by users with admin priviledges. euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` +The nova-manage command may only be run by users with admin priviledges. Commands for euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` User and Resource Management ---------------------------- -nova-manage and euca2ools provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. +The nova-manage and euca2ools commands provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. -For background on the core objects refenced in this section, see :doc:`../object.model` +For background on the core objects referenced in this section, see :doc:`../object.model` .. toctree:: :maxdepth: 1 diff --git a/doc/source/adminguide/managing.networks.rst b/doc/source/adminguide/managing.networks.rst index c8df471e84..b8563637e9 100644 --- a/doc/source/adminguide/managing.networks.rst +++ b/doc/source/adminguide/managing.networks.rst @@ -20,21 +20,6 @@ Networking Overview =================== In Nova, users organize their cloud resources in projects. A Nova project consists of a number of VM instances created by a user. For each VM instance, Nova assigns to it a private IP address. (Currently, Nova only supports Linux bridge networking that allows the virtual interfaces to connect to the outside network through the physical interface. Other virtual network technologies, such as Open vSwitch, could be supported in the future.) The Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. -.. - (perhaps some of this should be moved elsewhere) - Introduction - ------------ - - Nova consists of seven main components, with the Cloud Controller component representing the global state and interacting with all other components. API Server acts as the Web services front end for the cloud controller. Compute Controller provides compute server resources, and the Object Store component provides storage services. Auth Manager provides authentication and authorization services. Volume Controller provides fast and permanent block-level storage for the comput servers. Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. Scheduler selects the most suitable compute controller to host an instance. - - .. todo:: Insert Figure 1 image from "An OpenStack Network Overview" contributed by Citrix - - Nova is built on a shared-nothing, messaging-based architecture. All of the major components, that is Compute Controller, Volume Controller, Network Controller, and Object Store can be run on multiple servers. Cloud Controller communicates with Object Store via HTTP (Hyper Text Transfer Protocol), but it communicates with Scheduler, Network Controller, and Volume Controller via AMQP (Advanced Message Queue Protocol). To avoid blocking each component while waiting for a response, Nova uses asynchronous calls, with a call-back that gets triggered when a response is received. - - To achieve the shared-nothing property with multiple copies of the same component, Nova keeps all the cloud system state in a distributed data store. Updates to system state are written into this store, using atomic transactions when required. Requests for system state are read out of this store. In limited cases, the read results are cached within controllers for short periods of time (for example, the current list of system users.) - - .. note:: The database schema is available on the `OpenStack Wiki _`. - Nova Network Strategies ----------------------- diff --git a/doc/source/adminguide/network.vlan.rst b/doc/source/adminguide/network.vlan.rst index 5bbc54bed5..c6c4e7f91e 100644 --- a/doc/source/adminguide/network.vlan.rst +++ b/doc/source/adminguide/network.vlan.rst @@ -50,7 +50,7 @@ The following diagram illustrates how the communication that occurs between the Goals ----- -* each project is in a protected network segment +For our implementation of Nova, our goal is that each project is in a protected network segment. Here are the specifications we keep in mind for meeting this goal. * RFC-1918 IP space * public IP via NAT @@ -59,19 +59,19 @@ Goals * limited (project-admin controllable) access to other project segments * all connectivity to instance and cloud API is via VPN into the project segment -* common DMZ segment for support services (only visible from project segment) +We also keep as a goal a common DMZ segment for support services, meaning these items are only visible from project segment: * metadata * dashboard - Limitations ----------- +We kept in mind some of these limitations: + * Projects / cluster limited to available VLANs in switching infrastructure * Requires VPN for access to project segment - Implementation -------------- Currently Nova segregates project VLANs using 802.1q VLAN tagging in the diff --git a/doc/source/nova.concepts.rst b/doc/source/nova.concepts.rst index ddf0f1b829..d47438de71 100644 --- a/doc/source/nova.concepts.rst +++ b/doc/source/nova.concepts.rst @@ -23,13 +23,13 @@ Nova Concepts and Introduction Introduction ------------ -Nova is the software that controls your Infrastructure as as Service (IaaS) +Nova, also known as OpenStack Compute, is the software that controls your Infrastructure as as Service (IaaS) cloud computing platform. It is similar in scope to Amazon EC2 and Rackspace -CloudServers. Nova does not include any virtualization software, rather it +Cloud Servers. Nova does not include any virtualization software, rather it defines drivers that interact with underlying virtualization mechanisms that run on your host operating system, and exposes functionality over a web API. -This document does not attempt to explain fundamental concepts of cloud +This site does not attempt to explain fundamental concepts of cloud computing, IaaS, virtualization, or other related technologies. Instead, it focuses on describing how Nova's implementation of those concepts is achieved. @@ -64,6 +64,19 @@ Concept: Instances An 'instance' is a word for a virtual machine that runs inside the cloud. +Concept: System Architecture +---------------------------- + +Nova consists of seven main components, with the Cloud Controller component representing the global state and interacting with all other components. API Server acts as the Web services front end for the cloud controller. Compute Controller provides compute server resources, and the Object Store component provides storage services. Auth Manager provides authentication and authorization services. Volume Controller provides fast and permanent block-level storage for the comput servers. Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. Scheduler selects the most suitable compute controller to host an instance. + + .. image:: images/Novadiagram.png + +Nova is built on a shared-nothing, messaging-based architecture. All of the major components, that is Compute Controller, Volume Controller, Network Controller, and Object Store can be run on multiple servers. Cloud Controller communicates with Object Store via HTTP (Hyper Text Transfer Protocol), but it communicates with Scheduler, Network Controller, and Volume Controller via AMQP (Advanced Message Queue Protocol). To avoid blocking each component while waiting for a response, Nova uses asynchronous calls, with a call-back that gets triggered when a response is received. + +To achieve the shared-nothing property with multiple copies of the same component, Nova keeps all the cloud system state in a distributed data store. Updates to system state are written into this store, using atomic transactions when required. Requests for system state are read out of this store. In limited cases, the read results are cached within controllers for short periods of time (for example, the current list of system users.) + + .. note:: The database schema is available on the `OpenStack Wiki _`. + Concept: Storage ---------------- @@ -150,7 +163,7 @@ See doc:`nova.manage` in the Administration Guide for more details. Concept: Flags -------------- -python-gflags +Nova uses python-gflags for a distributed command line system, and the flags can either be set when running a command at the command line or within flag files. When you install Nova packages, each nova service gets its own flag file. For example, nova-network.conf is used for configuring the nova-network service, and so forth. Concept: Plugins From d6a74a219b8448d74a08730f6c452c20fb59abba Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 22 Nov 2010 17:03:54 -0600 Subject: [PATCH 010/119] Incorporating security groups info --- doc/source/devref/cloudpipe.rst | 2 +- doc/source/nova.concepts.rst | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/source/devref/cloudpipe.rst b/doc/source/devref/cloudpipe.rst index 31bd85e817..fb104c160e 100644 --- a/doc/source/devref/cloudpipe.rst +++ b/doc/source/devref/cloudpipe.rst @@ -21,7 +21,7 @@ Cloudpipe -- Per Project Vpns ============================= -Cloudpipe is a method for connecting end users to their project insnances in vlan mode. +Cloudpipe is a method for connecting end users to their project instances in vlan mode. Overview diff --git a/doc/source/nova.concepts.rst b/doc/source/nova.concepts.rst index d47438de71..18368546bd 100644 --- a/doc/source/nova.concepts.rst +++ b/doc/source/nova.concepts.rst @@ -117,9 +117,9 @@ Concept: API Concept: Networking ------------------- -Nova has a concept of Fixed Ips and Floating ips. Fixed ips are assigned to an instance on creation and stay the same until the instance is explicitly terminated. Floating ips are ip addresses that can be dynamically associated with an instance. This address can be disassociated and associated with another instance at any time. +Nova has a concept of Fixed IPs and Floating IPs. Fixed IPs are assigned to an instance on creation and stay the same until the instance is explicitly terminated. Floating ips are ip addresses that can be dynamically associated with an instance. This address can be disassociated and associated with another instance at any time. -There are multiple strategies available for implementing fixed ips: +There are multiple strategies available for implementing fixed IPs: Flat Mode ~~~~~~~~~ @@ -129,7 +129,7 @@ The simplest networking mode. Each instance receives a fixed ip from the pool. Flat DHCP Mode ~~~~~~~~~~~~~~ -This is similar to the flat mode, in that all instances are attached to the same bridge. In this mode nova does a bit more configuration, it will attempt to bridge into an ethernet device (eth0 by default). It will also run dnsmasq as a dhcpserver listening on this bridge. Instances receive their fixed ips by doing a dhcpdiscover. +This is similar to the flat mode, in that all instances are attached to the same bridge. In this mode nova does a bit more configuration, it will attempt to bridge into an ethernet device (eth0 by default). It will also run dnsmasq as a dhcpserver listening on this bridge. Instances receive their fixed IPs by doing a dhcpdiscover. VLAN DHCP Mode ~~~~~~~~~~~~~~ @@ -200,8 +200,17 @@ Concept: Scheduler Concept: Security Groups ------------------------ -Security groups +In Nova, a security group is a named collection of network access rules, like firewall policies. These access rules specify which incoming network traffic should be delivered to all VM instances in the group, all other incoming traffic being discarded. Users can modify rules for a group at any time. The new rules are automatically enforced for all running instances and instances launched from then on. +When launching VM instances, the project manager specifies which security groups it wants to join. It will become a member of these specified security groups when it is launched. If no groups are specified, the instances is assigned to the default group, which by default allows all network traffic from other members of this group and discards traffic from other IP addresses and groups. If this does not meet a user's needs, the user can modify the rule settings of the default group. + +A security group can be thought of as a security profile or a security role - it promotes the good practice of managing firewalls by role, not by machine. For example, a user could stipulate that servers with the "webapp" role must be able to connect to servers with the "mysql" role on port 3306. Going further with the security profile analogy, an instance can be launched with membership of multiple security groups - similar to a server with multiple roles. Because all rules in security groups are ACCEPT rules, it's trivial to combine them. + +Each rule in a security group must specify the source of packets to be allowed, which can either be a subnet anywhere on the Internet (in CIDR notation, with 0.0.0./0 representing the entire Internet) or another security group. In the latter case, the source security group can be any user's group. This makes it easy to grant selective access to one user's instances from instances run by the user's friends, partners, and vendors. + +The creation of rules with other security groups specified as sources helps users deal with dynamic IP addressing. Without this feature, the user would have had to adjust the security groups each time a new instance is launched. This practice would become cumbersome if an application running in Nova is very dynamic and elastic, for example scales up or down frequently. + +Security groups for a VM are passed at launch time by the cloud controller to the compute node, and applied at the compute node when a VM is started. Concept: Certificate Authority ------------------------------ From 309c8b8ff8732e8d80c445381aee7e1f9852def6 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 24 Nov 2010 22:10:21 +0000 Subject: [PATCH 011/119] Adding support for modification only of user accounts. --- nova/auth/ldapdriver.py | 110 ++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ceade1d65c..d1ef37cf0c 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,6 +40,8 @@ flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') +flags.DEFINE_boolean('ldap_user_modify_only', False, + 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -89,8 +91,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -110,7 +111,12 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - return [self.__to_user(attr) for attr in attrs] + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user != None: + users.append(user) + return users def get_projects(self, uid=None): """Retrieve list of projects""" @@ -125,21 +131,46 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + if FLAGS.ldap_user_modify_only: + if self.__ldap_user_exists(name): + # Retrieve user by name + user = self.__get_ldap_user(name) + if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): + raise exception.Duplicate("LDAP user %s already exists" % name) + else: + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) + else: + attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) + else: + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + ('uid', [name]), + ('sn', [name]), + ('cn', [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + ('isAdmin', [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -256,7 +287,21 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - self.conn.delete_s(self.__uid_to_dn(uid)) + if FLAGS.ldap_user_modify_only: + # Delete attributes + attr = [] + # Retrieve user by name + user = self.__get_ldap_user(uid) + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) + self.conn.modify_s(self.__uid_to_dn(uid), attr) + else: + # Delete entry + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -265,7 +310,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing project""" + """Modify an existing user""" if not access_key and not secret_key and admin is None: return attr = [] @@ -281,10 +326,20 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None + def __ldap_user_exists(self, uid): + """Check if the user exists in ldap""" + return self.__get_ldap_user(uid) != None + def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None + def __get_ldap_user(self, uid): + """Retrieve LDAP user entry by id""" + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') + return attr + def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" objects = self.__find_objects(dn, query, scope) @@ -449,12 +504,15 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr == None: return None - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} + if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): + return { + 'id': attr['uid'][0], + 'name': attr['uid'][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr['isAdmin'][0] == 'TRUE')} + else: + return None def __to_project(self, attr): """Convert ldap attributes to Project object""" From d7515bcb1d35e2e558a01c381b1d3a22165daa4b Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 24 Nov 2010 22:34:52 +0000 Subject: [PATCH 012/119] Setting "name" back to "cn", since id and name should be separate --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index d1ef37cf0c..95519d0005 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -507,7 +507,7 @@ class LdapDriver(object): if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): return { 'id': attr['uid'][0], - 'name': attr['uid'][0], + 'name': attr['cn'][0], 'access': attr['accessKey'][0], 'secret': attr['secretKey'][0], 'admin': (attr['isAdmin'][0] == 'TRUE')} From 1188dd95fbfef144ca71a3c9df2f7dbdb665c97f Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 24 Nov 2010 14:52:10 -0800 Subject: [PATCH 013/119] Consolidated the start instance logic in the two API classes into a single method. This also cleans up a number of small discrepencies between the two. --- nova/api/ec2/cloud.py | 174 ++++------------------- nova/api/openstack/servers.py | 96 ++----------- nova/compute/instance_types.py | 20 +++ nova/compute/manager.py | 130 +++++++++++++++++ nova/quota.py | 5 + nova/tests/api/openstack/fakes.py | 2 +- nova/tests/api/openstack/test_servers.py | 6 + nova/tests/quota_unittest.py | 16 ++- 8 files changed, 211 insertions(+), 238 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 9327bf0d44..c694579674 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -39,7 +39,7 @@ from nova import flags from nova import quota from nova import rpc from nova import utils -from nova.compute.instance_types import INSTANCE_TYPES +from nova.compute import instance_types from nova.api import cloud from nova.image.s3 import S3ImageService @@ -50,11 +50,6 @@ flags.DECLARE('storage_availability_zone', 'nova.volume.manager') InvalidInputException = exception.InvalidInputException -class QuotaError(exception.ApiError): - """Quota Exceeeded""" - pass - - def _gen_key(context, user_id, key_name): """Generate a key @@ -127,7 +122,7 @@ class CloudController(object): for instance in db.instance_get_all_by_project(context, project_id): if instance['fixed_ip']: line = '%s slots=%d' % (instance['fixed_ip']['address'], - INSTANCE_TYPES[instance['instance_type']]['vcpus']) + instance['vcpus']) key = str(instance['key_name']) if key in result: result[key].append(line) @@ -260,7 +255,7 @@ class CloudController(object): return True def describe_security_groups(self, context, group_name=None, **kwargs): - self._ensure_default_security_group(context) + self.compute_manager.ensure_default_security_group(context) if context.user.is_admin(): groups = db.security_group_get_all(context) else: @@ -358,7 +353,7 @@ class CloudController(object): return False def revoke_security_group_ingress(self, context, group_name, **kwargs): - self._ensure_default_security_group(context) + self.compute_manager.ensure_default_security_group(context) security_group = db.security_group_get_by_name(context, context.project_id, group_name) @@ -383,7 +378,7 @@ class CloudController(object): # for these operations, so support for newer API versions # is sketchy. def authorize_security_group_ingress(self, context, group_name, **kwargs): - self._ensure_default_security_group(context) + self.compute_manager.ensure_default_security_group(context) security_group = db.security_group_get_by_name(context, context.project_id, group_name) @@ -419,7 +414,7 @@ class CloudController(object): return source_project_id def create_security_group(self, context, group_name, group_description): - self._ensure_default_security_group(context) + self.compute_manager.ensure_default_security_group(context) if db.security_group_exists(context, context.project_id, group_name): raise exception.ApiError('group %s already exists' % group_name) @@ -505,9 +500,8 @@ class CloudController(object): if quota.allowed_volumes(context, 1, size) < 1: logging.warn("Quota exceeeded for %s, tried to create %sG volume", context.project_id, size) - raise QuotaError("Volume quota exceeded. You cannot " - "create a volume of size %s" % - size) + raise quota.QuotaError("Volume quota exceeded. You cannot " + "create a volume of size %s" % size) vol = {} vol['size'] = size vol['user_id'] = context.user.id @@ -699,8 +693,8 @@ class CloudController(object): if quota.allowed_floating_ips(context, 1) < 1: logging.warn("Quota exceeeded for %s, tried to allocate address", context.project_id) - raise QuotaError("Address quota exceeded. You cannot " - "allocate any more addresses") + raise quota.QuotaError("Address quota exceeded. You cannot " + "allocate any more addresses") network_topic = self._get_network_topic(context) public_ip = rpc.call(context, network_topic, @@ -752,137 +746,25 @@ class CloudController(object): "args": {"network_id": network_ref['id']}}) return db.queue_get_for(context, FLAGS.network_topic, host) - def _ensure_default_security_group(self, context): - try: - db.security_group_get_by_name(context, - context.project_id, - 'default') - except exception.NotFound: - values = {'name': 'default', - 'description': 'default', - 'user_id': context.user.id, - 'project_id': context.project_id} - group = db.security_group_create(context, values) - def run_instances(self, context, **kwargs): - instance_type = kwargs.get('instance_type', 'm1.small') - if instance_type not in INSTANCE_TYPES: - raise exception.ApiError("Unknown instance type: %s", - instance_type) - # check quota - max_instances = int(kwargs.get('max_count', 1)) - min_instances = int(kwargs.get('min_count', max_instances)) - num_instances = quota.allowed_instances(context, - max_instances, - instance_type) - if num_instances < min_instances: - logging.warn("Quota exceeeded for %s, tried to run %s instances", - context.project_id, min_instances) - raise QuotaError("Instance quota exceeded. You can only " - "run %s more instances of this type." % - num_instances, "InstanceLimitExceeded") - # make sure user can access the image - # vpn image is private so it doesn't show up on lists - vpn = kwargs['image_id'] == FLAGS.vpn_image_id - - if not vpn: - image = self.image_service.show(context, kwargs['image_id']) - - # FIXME(ja): if image is vpn, this breaks - # get defaults from imagestore - image_id = image['imageId'] - kernel_id = image.get('kernelId', FLAGS.default_kernel) - ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) - - # API parameters overrides of defaults - kernel_id = kwargs.get('kernel_id', kernel_id) - ramdisk_id = kwargs.get('ramdisk_id', ramdisk_id) - - # make sure we have access to kernel and ramdisk - self.image_service.show(context, kernel_id) - self.image_service.show(context, ramdisk_id) - - logging.debug("Going to run %s instances...", num_instances) - launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) - key_data = None - if 'key_name' in kwargs: - key_pair_ref = db.key_pair_get(context, - context.user.id, - kwargs['key_name']) - key_data = key_pair_ref['public_key'] - - security_group_arg = kwargs.get('security_group', ["default"]) - if not type(security_group_arg) is list: - security_group_arg = [security_group_arg] - - security_groups = [] - self._ensure_default_security_group(context) - for security_group_name in security_group_arg: - group = db.security_group_get_by_name(context, - context.project_id, - security_group_name) - security_groups.append(group['id']) - - reservation_id = utils.generate_uid('r') - base_options = {} - base_options['state_description'] = 'scheduling' - base_options['image_id'] = image_id - base_options['kernel_id'] = kernel_id - base_options['ramdisk_id'] = ramdisk_id - base_options['reservation_id'] = reservation_id - base_options['key_data'] = key_data - base_options['key_name'] = kwargs.get('key_name', None) - base_options['user_id'] = context.user.id - base_options['project_id'] = context.project_id - base_options['user_data'] = kwargs.get('user_data', '') - - base_options['display_name'] = kwargs.get('display_name') - base_options['display_description'] = kwargs.get('display_description') - - type_data = INSTANCE_TYPES[instance_type] - base_options['instance_type'] = instance_type - base_options['memory_mb'] = type_data['memory_mb'] - base_options['vcpus'] = type_data['vcpus'] - base_options['local_gb'] = type_data['local_gb'] - elevated = context.elevated() - - for num in range(num_instances): - - instance_ref = self.compute_manager.create_instance(context, - security_groups, - mac_address=utils.generate_mac(), - launch_index=num, - **base_options) - inst_id = instance_ref['id'] - - internal_id = instance_ref['internal_id'] - ec2_id = internal_id_to_ec2_id(internal_id) - - self.compute_manager.update_instance(context, - inst_id, - hostname=ec2_id) - - # TODO(vish): This probably should be done in the scheduler - # or in compute as a call. The network should be - # allocated after the host is assigned and setup - # can happen at the same time. - address = self.network_manager.allocate_fixed_ip(context, - inst_id, - vpn) - network_topic = self._get_network_topic(context) - rpc.cast(elevated, - network_topic, - {"method": "setup_fixed_ip", - "args": {"address": address}}) - - rpc.cast(context, - FLAGS.scheduler_topic, - {"method": "run_instance", - "args": {"topic": FLAGS.compute_topic, - "instance_id": inst_id}}) - logging.debug("Casting to scheduler for %s/%s's instance %s" % - (context.project.name, context.user.name, inst_id)) - return self._format_run_instances(context, reservation_id) + max_count = int(kwargs.get('max_count', 1)) + instances = self.compute_manager.create_instances(context, + instance_types.get_by_type(kwargs.get('instance_type', None)), + self.image_service, + kwargs['image_id'], + self._get_network_topic(context), + min_count=int(kwargs.get('min_count', max_count)), + max_count=max_count, + kernel_id=kwargs.get('kernel_id'), + ramdisk_id=kwargs.get('ramdisk_id'), + name=kwargs.get('display_name'), + description=kwargs.get('display_description'), + user_data=kwargs.get('user_data', ''), + key_name=kwargs.get('key_name'), + security_group=kwargs.get('security_group'), + generate_hostname=internal_id_to_ec2_id) + return self._format_run_instances(context, + instances[0]['reservation_id']) def terminate_instances(self, context, instance_id, **kwargs): """Terminate each instance in instance_id, which is a list of ec2 ids. diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 1d8aa2fa4d..e1e2bf7fd4 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -140,22 +140,23 @@ class Controller(wsgi.Controller): def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - #try: - inst = self._build_server_instance(req, env) - #except Exception, e: - # return faults.Fault(exc.HTTPUnprocessableEntity()) - user_id = req.environ['nova.context']['user']['id'] - rpc.cast(context.RequestContext(user_id, user_id), - FLAGS.compute_topic, - {"method": "run_instance", - "args": {"instance_id": inst['id']}}) - return _entity_inst(inst) + ctxt = context.RequestContext(user_id, user_id) + key_pair = self.db_driver.key_pair_get_all_by_user(None, user_id)[0] + instances = self.compute_manager.create_instances(ctxt, + instance_types.get_by_flavor_id(env['server']['flavorId']), + utils.import_object(FLAGS.image_service), + env['server']['imageId'], + self._get_network_topic(ctxt), + name=env['server']['name'], + description=env['server']['name'], + key_name=key_pair['name'], + key_data=key_pair['public_key']) + return _entity_inst(instances[0]) def update(self, req, id): """ Updates the server name or password """ @@ -191,79 +192,6 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) cloud.reboot(id) - def _build_server_instance(self, req, env): - """Build instance data structure and save it to the data store.""" - ltime = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) - inst = {} - - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - - flavor_id = env['server']['flavorId'] - - instance_type, flavor = [(k, v) for k, v in - instance_types.INSTANCE_TYPES.iteritems() - if v['flavorid'] == flavor_id][0] - - image_id = env['server']['imageId'] - img_service = utils.import_object(FLAGS.image_service) - - image = img_service.show(image_id) - - if not image: - raise Exception("Image not found") - - inst['server_name'] = env['server']['name'] - inst['image_id'] = image_id - inst['user_id'] = user_id - inst['launch_time'] = ltime - inst['mac_address'] = utils.generate_mac() - inst['project_id'] = user_id - - inst['state_description'] = 'scheduling' - inst['kernel_id'] = image.get('kernelId', FLAGS.default_kernel) - inst['ramdisk_id'] = image.get('ramdiskId', FLAGS.default_ramdisk) - inst['reservation_id'] = utils.generate_uid('r') - - inst['display_name'] = env['server']['name'] - inst['display_description'] = env['server']['name'] - - #TODO(dietz) this may be ill advised - key_pair_ref = self.db_driver.key_pair_get_all_by_user( - None, user_id)[0] - - inst['key_data'] = key_pair_ref['public_key'] - inst['key_name'] = key_pair_ref['name'] - - #TODO(dietz) stolen from ec2 api, see TODO there - inst['security_group'] = 'default' - - # Flavor related attributes - inst['instance_type'] = instance_type - inst['memory_mb'] = flavor['memory_mb'] - inst['vcpus'] = flavor['vcpus'] - inst['local_gb'] = flavor['local_gb'] - inst['mac_address'] = utils.generate_mac() - inst['launch_index'] = 0 - - ref = self.compute_manager.create_instance(ctxt, **inst) - inst['id'] = ref['internal_id'] - - inst['hostname'] = str(ref['internal_id']) - self.compute_manager.update_instance(ctxt, inst['id'], **inst) - - address = self.network_manager.allocate_fixed_ip(ctxt, - inst['id']) - - # TODO(vish): This probably should be done in the scheduler - # network is setup when host is assigned - network_topic = self._get_network_topic(ctxt) - rpc.call(ctxt, - network_topic, - {"method": "setup_fixed_ip", - "args": {"address": address}}) - return inst - def _get_network_topic(self, context): """Retrieves the network host for a project""" network_ref = self.network_manager.get_network(context) diff --git a/nova/compute/instance_types.py b/nova/compute/instance_types.py index 67ee8f8a8a..a2679e0fc2 100644 --- a/nova/compute/instance_types.py +++ b/nova/compute/instance_types.py @@ -21,9 +21,29 @@ The built-in instance properties. """ +from nova import flags + +FLAGS = flags.FLAGS INSTANCE_TYPES = { 'm1.tiny': dict(memory_mb=512, vcpus=1, local_gb=0, flavorid=1), 'm1.small': dict(memory_mb=2048, vcpus=1, local_gb=20, flavorid=2), 'm1.medium': dict(memory_mb=4096, vcpus=2, local_gb=40, flavorid=3), 'm1.large': dict(memory_mb=8192, vcpus=4, local_gb=80, flavorid=4), 'm1.xlarge': dict(memory_mb=16384, vcpus=8, local_gb=160, flavorid=5)} + + +def get_by_type(instance_type): + """Build instance data structure and save it to the data store.""" + if instance_type is None: + return FLAGS.default_instance_type + if instance_type not in INSTANCE_TYPES: + raise exception.ApiError("Unknown instance type: %s", + instance_type) + return instance_type + + +def get_by_flavor_id(flavor_id): + for instance_type, details in INSTANCE_TYPES.iteritems(): + if details['flavorid'] == flavor_id: + return instance_type + return FLAGS.default_instance_type diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 890d79fba9..cfc3d4bbd9 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -36,13 +36,18 @@ termination. import datetime import logging +import time from twisted.internet import defer +from nova import db from nova import exception from nova import flags from nova import manager +from nova import quota +from nova import rpc from nova import utils +from nova.compute import instance_types from nova.compute import power_state @@ -53,6 +58,11 @@ flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', 'Driver to use for volume creation') +def generate_default_hostname(internal_id): + """Default function to generate a hostname given an instance reference.""" + return str(internal_id) + + class ComputeManager(manager.Manager): """Manages the running instances from creation to destruction.""" @@ -84,6 +94,126 @@ class ComputeManager(manager.Manager): """This call passes stright through to the virtualization driver.""" yield self.driver.refresh_security_group(security_group_id) + # TODO(eday): network_topic arg should go away once we push network + # allocation into the scheduler or compute worker. + def create_instances(self, context, instance_type, image_service, image_id, + network_topic, min_count=1, max_count=1, + kernel_id=None, ramdisk_id=None, name='', + description='', user_data='', key_name=None, + key_data=None, security_group='default', + generate_hostname=generate_default_hostname): + """Create the number of instances requested if quote and + other arguments check out ok.""" + + num_instances = quota.allowed_instances(context, max_count, + instance_type) + if num_instances < min_count: + logging.warn("Quota exceeeded for %s, tried to run %s instances", + context.project_id, min_count) + raise quota.QuotaError("Instance quota exceeded. You can only " + "run %s more instances of this type." % + num_instances, "InstanceLimitExceeded") + + is_vpn = image_id == FLAGS.vpn_image_id + if not is_vpn: + image = image_service.show(context, image_id) + if not image: + raise Exception("Image not found") + if kernel_id is None: + kernel_id = image.get('kernelId', FLAGS.default_kernel) + if ramdisk_id is None: + ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) + + # Make sure we have access to kernel and ramdisk + image_service.show(context, kernel_id) + image_service.show(context, ramdisk_id) + + if security_group is None: + security_group = ['default'] + if not type(security_group) is list: + security_group = [security_group] + + security_groups = [] + self.ensure_default_security_group(context) + for security_group_name in security_group: + group = db.security_group_get_by_name(context, + context.project_id, + security_group_name) + security_groups.append(group['id']) + + if key_data is None and key_name: + key_pair = db.key_pair_get(context, context.user_id, key_name) + key_data = key_pair['public_key'] + + type_data = instance_types.INSTANCE_TYPES[instance_type] + base_options = { + 'reservation_id': utils.generate_uid('r'), + 'server_name': name, + 'image_id': image_id, + 'kernel_id': kernel_id, + 'ramdisk_id': ramdisk_id, + 'state_description': 'scheduling', + 'user_id': context.user_id, + 'project_id': context.project_id, + 'launch_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'instance_type': instance_type, + 'memory_mb': type_data['memory_mb'], + 'vcpus': type_data['vcpus'], + 'local_gb': type_data['local_gb'], + 'display_name': name, + 'display_description': description, + 'key_name': key_name, + 'key_data': key_data} + + elevated = context.elevated() + instances = [] + logging.debug("Going to run %s instances...", num_instances) + for num in range(num_instances): + instance = dict(mac_address=utils.generate_mac(), + launch_index=num, + **base_options) + instance_ref = self.create_instance(context, security_groups, + **instance) + instance_id = instance_ref['id'] + internal_id = instance_ref['internal_id'] + hostname = generate_hostname(internal_id) + self.update_instance(context, instance_id, hostname=hostname) + instances.append(dict(id=instance_id, internal_id=internal_id, + hostname=hostname, **instance)) + + # TODO(vish): This probably should be done in the scheduler + # or in compute as a call. The network should be + # allocated after the host is assigned and setup + # can happen at the same time. + address = self.network_manager.allocate_fixed_ip(context, + instance_id, + is_vpn) + rpc.cast(elevated, + network_topic, + {"method": "setup_fixed_ip", + "args": {"address": address}}) + + logging.debug("Casting to scheduler for %s/%s's instance %s" % + (context.project_id, context.user_id, instance_id)) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "run_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id}}) + + return instances + + def ensure_default_security_group(self, context): + try: + db.security_group_get_by_name(context, context.project_id, + 'default') + except exception.NotFound: + values = {'name': 'default', + 'description': 'default', + 'user_id': context.user_id, + 'project_id': context.project_id} + group = db.security_group_create(context, values) + def create_instance(self, context, security_groups=None, **kwargs): """Creates the instance in the datastore and returns the new instance as a mapping diff --git a/nova/quota.py b/nova/quota.py index 01dd0ecd45..f6ca9f77c7 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -94,3 +94,8 @@ def allowed_floating_ips(context, num_floating_ips): quota = get_quota(context, project_id) allowed_floating_ips = quota['floating_ips'] - used_floating_ips return min(num_floating_ips, allowed_floating_ips) + + +class QuotaError(exception.ApiError): + """Quota Exceeeded""" + pass diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 639a2ebe4e..e819fbc175 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -73,7 +73,7 @@ def stub_out_key_pair_funcs(stubs): def stub_out_image_service(stubs): - def fake_image_show(meh, id): + def fake_image_show(meh, context, id): return dict(kernelId=1, ramdiskId=1) stubs.Set(nova.image.local.LocalImageService, 'show', fake_image_show) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 8cfc6c45a3..0d540c0375 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -43,6 +43,10 @@ def return_servers(context, user_id=1): return [stub_instance(i, user_id) for i in xrange(5)] +def return_security_group(context, instance_id, security_group_id): + pass + + def stub_instance(id, user_id=1): return Instance(id=id, state=0, image_id=10, server_name='server%s' % id, user_id=user_id) @@ -63,6 +67,8 @@ class ServersTest(unittest.TestCase): return_server) self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers) + self.stubs.Set(nova.db.api, 'instance_add_security_group', + return_security_group) def tearDown(self): self.stubs.UnsetAll() diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index b7c1d2acc0..1966b51f7b 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -94,11 +94,12 @@ class QuotaTestCase(test.TrialTestCase): for i in range(FLAGS.quota_instances): instance_id = self._create_instance() instance_ids.append(instance_id) - self.assertRaises(cloud.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, self.cloud.run_instances, self.context, min_count=1, max_count=1, - instance_type='m1.small') + instance_type='m1.small', + image_id='fake') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -106,11 +107,12 @@ class QuotaTestCase(test.TrialTestCase): instance_ids = [] instance_id = self._create_instance(cores=4) instance_ids.append(instance_id) - self.assertRaises(cloud.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, self.cloud.run_instances, self.context, min_count=1, max_count=1, - instance_type='m1.small') + instance_type='m1.small', + image_id='fake') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -119,7 +121,7 @@ class QuotaTestCase(test.TrialTestCase): for i in range(FLAGS.quota_volumes): volume_id = self._create_volume() volume_ids.append(volume_id) - self.assertRaises(cloud.QuotaError, self.cloud.create_volume, + self.assertRaises(quota.QuotaError, self.cloud.create_volume, self.context, size=10) for volume_id in volume_ids: @@ -129,7 +131,7 @@ class QuotaTestCase(test.TrialTestCase): volume_ids = [] volume_id = self._create_volume(size=20) volume_ids.append(volume_id) - self.assertRaises(cloud.QuotaError, + self.assertRaises(quota.QuotaError, self.cloud.create_volume, self.context, size=10) @@ -146,6 +148,6 @@ class QuotaTestCase(test.TrialTestCase): # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertRaises(cloud.QuotaError, self.cloud.allocate_address, + self.assertRaises(quota.QuotaError, self.cloud.allocate_address, self.context) db.floating_ip_destroy(context.get_admin_context(), address) From 7d771bf9c549499c0a138ea991da5df537e0dd88 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 24 Nov 2010 15:16:23 -0800 Subject: [PATCH 014/119] The image server should throw not found errors, don't need to check in compute manager. --- nova/compute/manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index cfc3d4bbd9..3f870f8666 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -117,8 +117,6 @@ class ComputeManager(manager.Manager): is_vpn = image_id == FLAGS.vpn_image_id if not is_vpn: image = image_service.show(context, image_id) - if not image: - raise Exception("Image not found") if kernel_id is None: kernel_id = image.get('kernelId', FLAGS.default_kernel) if ramdisk_id is None: From 725a1f638b01985a2ae9a4f0a68f16ef31914a51 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 26 Nov 2010 17:01:50 +0000 Subject: [PATCH 015/119] This modification should have occured in a different branch. Reverting. --- nova/auth/ldapdriver.py | 110 ++++++++++------------------------------ 1 file changed, 26 insertions(+), 84 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 95519d0005..ceade1d65c 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,8 +40,6 @@ flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') -flags.DEFINE_boolean('ldap_user_modify_only', False, - 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -91,7 +89,8 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__get_ldap_user(uid) + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -111,12 +110,7 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - users = [] - for attr in attrs: - user = self.__to_user(attr) - if user != None: - users.append(user) - return users + return [self.__to_user(attr) for attr in attrs] def get_projects(self, uid=None): """Retrieve list of projects""" @@ -131,46 +125,21 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - if FLAGS.ldap_user_modify_only: - if self.__ldap_user_exists(name): - # Retrieve user by name - user = self.__get_ldap_user(name) - if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): - raise exception.Duplicate("LDAP user %s already exists" % name) - else: - # Entry could be malformed, test for missing attrs. - # Malformed entries are useless, replace attributes found. - attr = [] - if user.has_key('secretKey'): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) - else: - attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) - if user.has_key('accessKey'): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) - else: - attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) - if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) - else: - attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) - self.conn.modify_s(self.__uid_to_dn(name), attr) - return self.get_user(name) - else: - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + ('uid', [name]), + ('sn', [name]), + ('cn', [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + ('isAdmin', [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -287,21 +256,7 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - if FLAGS.ldap_user_modify_only: - # Delete attributes - attr = [] - # Retrieve user by name - user = self.__get_ldap_user(uid) - if user.has_key('secretKey'): - attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) - if user.has_key('accessKey'): - attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) - if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) - self.conn.modify_s(self.__uid_to_dn(uid), attr) - else: - # Delete entry - self.conn.delete_s(self.__uid_to_dn(uid)) + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -310,7 +265,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing user""" + """Modify an existing project""" if not access_key and not secret_key and admin is None: return attr = [] @@ -326,20 +281,10 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None - def __ldap_user_exists(self, uid): - """Check if the user exists in ldap""" - return self.__get_ldap_user(uid) != None - def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None - def __get_ldap_user(self, uid): - """Retrieve LDAP user entry by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') - return attr - def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" objects = self.__find_objects(dn, query, scope) @@ -504,15 +449,12 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr == None: return None - if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} - else: - return None + return { + 'id': attr['uid'][0], + 'name': attr['cn'][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr['isAdmin'][0] == 'TRUE')} def __to_project(self, attr): """Convert ldap attributes to Project object""" From c3072aea3dc5d44d26fcac5c7db65b8cc445fccc Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 26 Nov 2010 17:04:27 +0000 Subject: [PATCH 016/119] Adding support for modification only of user accounts. --- nova/auth/ldapdriver.py | 110 ++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ceade1d65c..95519d0005 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,6 +40,8 @@ flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') +flags.DEFINE_boolean('ldap_user_modify_only', False, + 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -89,8 +91,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -110,7 +111,12 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - return [self.__to_user(attr) for attr in attrs] + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user != None: + users.append(user) + return users def get_projects(self, uid=None): """Retrieve list of projects""" @@ -125,21 +131,46 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + if FLAGS.ldap_user_modify_only: + if self.__ldap_user_exists(name): + # Retrieve user by name + user = self.__get_ldap_user(name) + if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): + raise exception.Duplicate("LDAP user %s already exists" % name) + else: + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) + else: + attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) + else: + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + ('uid', [name]), + ('sn', [name]), + ('cn', [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + ('isAdmin', [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -256,7 +287,21 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - self.conn.delete_s(self.__uid_to_dn(uid)) + if FLAGS.ldap_user_modify_only: + # Delete attributes + attr = [] + # Retrieve user by name + user = self.__get_ldap_user(uid) + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) + self.conn.modify_s(self.__uid_to_dn(uid), attr) + else: + # Delete entry + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -265,7 +310,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing project""" + """Modify an existing user""" if not access_key and not secret_key and admin is None: return attr = [] @@ -281,10 +326,20 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None + def __ldap_user_exists(self, uid): + """Check if the user exists in ldap""" + return self.__get_ldap_user(uid) != None + def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None + def __get_ldap_user(self, uid): + """Retrieve LDAP user entry by id""" + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') + return attr + def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" objects = self.__find_objects(dn, query, scope) @@ -449,12 +504,15 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr == None: return None - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} + if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): + return { + 'id': attr['uid'][0], + 'name': attr['cn'][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr['isAdmin'][0] == 'TRUE')} + else: + return None def __to_project(self, attr): """Convert ldap attributes to Project object""" From 8a7e6e0f003e1b3837b918ac9af1564ac1665aae Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 26 Nov 2010 17:59:48 +0000 Subject: [PATCH 017/119] PEP fixes --- nova/auth/ldapdriver.py | 72 ++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 95519d0005..fa48c8435f 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -91,7 +91,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__get_ldap_user(uid) + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -111,11 +111,11 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - users = [] - for attr in attrs: - user = self.__to_user(attr) - if user != None: - users.append(user) + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user is not None: + users.append(user) return users def get_projects(self, uid=None): @@ -135,24 +135,32 @@ class LdapDriver(object): if self.__ldap_user_exists(name): # Retrieve user by name user = self.__get_ldap_user(name) - if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): - raise exception.Duplicate("LDAP user %s already exists" % name) + if user.has_key('accessKey') and user.has_key('secretKey') \ + and user.has_key('isAdmin'): + raise exception.Duplicate("LDAP user %s already exists" \ + % name) else: # Entry could be malformed, test for missing attrs. # Malformed entries are useless, replace attributes found. attr = [] if user.has_key('secretKey'): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) + attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ + [secret_key])) else: - attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) + attr.append((self.ldap.MOD_ADD, 'secretKey', \ + [secret_key])) if user.has_key('accessKey'): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) + attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ + [access_key])) else: - attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) + attr.append((self.ldap.MOD_ADD, 'accessKey', \ + [access_key])) if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ + [str(is_admin).upper()])) else: - attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) + attr.append((self.ldap.MOD_ADD, 'isAdmin', \ + [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: @@ -186,7 +194,7 @@ class LdapDriver(object): if description is None: description = name members = [] - if member_uids != None: + if member_uids is not None: for member_uid in member_uids: if not self.__user_exists(member_uid): raise exception.NotFound("Project can't be created " @@ -293,11 +301,14 @@ class LdapDriver(object): # Retrieve user by name user = self.__get_ldap_user(uid) if user.has_key('secretKey'): - attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) + attr.append((self.ldap.MOD_DELETE, 'secretKey', \ + user['secretKey'])) if user.has_key('accessKey'): - attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) + attr.append((self.ldap.MOD_DELETE, 'accessKey', \ + user['accessKey'])) if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) + attr.append((self.ldap.MOD_DELETE, 'isAdmin', \ + user['isAdmin'])) self.conn.modify_s(self.__uid_to_dn(uid), attr) else: # Delete entry @@ -324,18 +335,18 @@ class LdapDriver(object): def __user_exists(self, uid): """Check if user exists""" - return self.get_user(uid) != None + return self.get_user(uid) is not None def __ldap_user_exists(self, uid): """Check if the user exists in ldap""" - return self.__get_ldap_user(uid) != None + return self.__get_ldap_user(uid) is not None def __project_exists(self, project_id): """Check if project exists""" - return self.get_project(project_id) != None + return self.get_project(project_id) is not None def __get_ldap_user(self, uid): - """Retrieve LDAP user entry by id""" + """Retrieve LDAP user entry by id""" attr = self.__find_object(self.__uid_to_dn(uid), '(objectclass=novaUser)') return attr @@ -385,12 +396,12 @@ class LdapDriver(object): def __group_exists(self, dn): """Check if group exists""" - return self.__find_object(dn, '(objectclass=groupOfNames)') != None + return self.__find_object(dn, '(objectclass=groupOfNames)') is not None @staticmethod def __role_to_dn(role, project_id=None): """Convert role to corresponding dn""" - if project_id == None: + if project_id is None: return FLAGS.__getitem__("ldap_%s" % role).value else: return 'cn=%s,cn=%s,%s' % (role, @@ -404,7 +415,7 @@ class LdapDriver(object): raise exception.Duplicate("Group can't be created because " "group %s already exists" % name) members = [] - if member_uids != None: + if member_uids is not None: for member_uid in member_uids: if not self.__user_exists(member_uid): raise exception.NotFound("Group can't be created " @@ -430,7 +441,7 @@ class LdapDriver(object): res = self.__find_object(group_dn, '(member=%s)' % self.__uid_to_dn(uid), self.ldap.SCOPE_BASE) - return res != None + return res is not None def __add_to_group(self, uid, group_dn): """Add user to group""" @@ -502,21 +513,22 @@ class LdapDriver(object): @staticmethod def __to_user(attr): """Convert ldap attributes to User object""" - if attr == None: + if attr is None: return None - if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): + if (attr.has_key('accessKey') and attr.has_key('secretKey') \ + and attr.has_key('isAdmin')): return { 'id': attr['uid'][0], 'name': attr['cn'][0], 'access': attr['accessKey'][0], 'secret': attr['secretKey'][0], 'admin': (attr['isAdmin'][0] == 'TRUE')} - else: + else: return None def __to_project(self, attr): """Convert ldap attributes to Project object""" - if attr == None: + if attr is None: return None member_dns = attr.get('member', []) return { From a44ee54dfe3f243a44636e9224082e86fdee452f Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Sat, 27 Nov 2010 12:56:19 +0000 Subject: [PATCH 018/119] first cut of the refactoring of the XenAPIConnection class. Currently the class merged both the code for managing the XenAPI connection and the business logic for implementing Nova operations. If left like this, it would eventually become difficult to read, maintain and extend. The file was getting kind of big and cluttered, so a quick refactoring now will save a lot of headaches later. --- nova/virt/xenapi.py | 321 ++++------------------- nova/virt/xenapi/power_state.py | 26 ++ nova/virt/xenapi/xenapi.py | 439 ++++++++++++++++++++++++++++++++ 3 files changed, 514 insertions(+), 272 deletions(-) create mode 100644 nova/virt/xenapi/power_state.py create mode 100644 nova/virt/xenapi/xenapi.py diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 3169562a53..93c1192056 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -54,14 +54,9 @@ from twisted.internet import defer from twisted.internet import reactor from twisted.internet import task -from nova import db -from nova import flags -from nova import process -from nova import utils -from nova.auth.manager import AuthManager -from nova.compute import instance_types -from nova.compute import power_state -from nova.virt import images +from xenapi import power_state +from xenapi import vmops +from xenapi import volumeops XenAPI = None @@ -86,14 +81,6 @@ flags.DEFINE_float('xenapi_task_poll_interval', 'connection_type=xenapi.') -XENAPI_POWER_STATE = { - 'Halted': power_state.SHUTDOWN, - 'Running': power_state.RUNNING, - 'Paused': power_state.PAUSED, - 'Suspended': power_state.SHUTDOWN, # FIXME - 'Crashed': power_state.CRASHED} - - def get_connection(_): """Note that XenAPI doesn't have a read-only connection mode, so the read_only parameter is ignored.""" @@ -115,273 +102,83 @@ def get_connection(_): class XenAPIConnection(object): def __init__(self, url, user, pw): - self._conn = XenAPI.Session(url) - self._conn.login_with_password(user, pw) - + self._session = XenAPISession(url, user, pw) + self._vmops = VMOps(sef._session) + self._volumeops = volumeOps(self._session) + def list_instances(self): - return [self._conn.xenapi.VM.get_name_label(vm) \ - for vm in self._conn.xenapi.VM.get_all()] - - @defer.inlineCallbacks + return self._vmops.list_instances() + def spawn(self, instance): - vm = yield self._lookup(instance.name) - if vm is not None: - raise Exception('Attempted to create non-unique name %s' % - instance.name) - - network = db.project_get_network(None, instance.project_id) - network_ref = \ - yield self._find_network_with_bridge(network.bridge) - - user = AuthManager().get_user(instance.user_id) - project = AuthManager().get_project(instance.project_id) - vdi_uuid = yield self._fetch_image( - instance.image_id, user, project, True) - kernel = yield self._fetch_image( - instance.kernel_id, user, project, False) - ramdisk = yield self._fetch_image( - instance.ramdisk_id, user, project, False) - vdi_ref = yield self._call_xenapi('VDI.get_by_uuid', vdi_uuid) - - vm_ref = yield self._create_vm(instance, kernel, ramdisk) - yield self._create_vbd(vm_ref, vdi_ref, 0, True) - if network_ref: - yield self._create_vif(vm_ref, network_ref, instance.mac_address) - logging.debug('Starting VM %s...', vm_ref) - yield self._call_xenapi('VM.start', vm_ref, False, False) - logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - - @defer.inlineCallbacks - def _create_vm(self, instance, kernel, ramdisk): - """Create a VM record. Returns a Deferred that gives the new - VM reference.""" - - instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] - mem = str(long(instance_type['memory_mb']) * 1024 * 1024) - vcpus = str(instance_type['vcpus']) - rec = { - 'name_label': instance.name, - 'name_description': '', - 'is_a_template': False, - 'memory_static_min': '0', - 'memory_static_max': mem, - 'memory_dynamic_min': mem, - 'memory_dynamic_max': mem, - 'VCPUs_at_startup': vcpus, - 'VCPUs_max': vcpus, - 'VCPUs_params': {}, - 'actions_after_shutdown': 'destroy', - 'actions_after_reboot': 'restart', - 'actions_after_crash': 'destroy', - 'PV_bootloader': '', - 'PV_kernel': kernel, - 'PV_ramdisk': ramdisk, - 'PV_args': 'root=/dev/xvda1', - 'PV_bootloader_args': '', - 'PV_legacy_args': '', - 'HVM_boot_policy': '', - 'HVM_boot_params': {}, - 'platform': {}, - 'PCI_bus': '', - 'recommendations': '', - 'affinity': '', - 'user_version': '0', - 'other_config': {}, - } - logging.debug('Created VM %s...', instance.name) - vm_ref = yield self._call_xenapi('VM.create', rec) - logging.debug('Created VM %s as %s.', instance.name, vm_ref) - defer.returnValue(vm_ref) - - @defer.inlineCallbacks - def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): - """Create a VBD record. Returns a Deferred that gives the new - VBD reference.""" - - vbd_rec = {} - vbd_rec['VM'] = vm_ref - vbd_rec['VDI'] = vdi_ref - vbd_rec['userdevice'] = str(userdevice) - vbd_rec['bootable'] = bootable - vbd_rec['mode'] = 'RW' - vbd_rec['type'] = 'disk' - vbd_rec['unpluggable'] = True - vbd_rec['empty'] = False - vbd_rec['other_config'] = {} - vbd_rec['qos_algorithm_type'] = '' - vbd_rec['qos_algorithm_params'] = {} - vbd_rec['qos_supported_algorithms'] = [] - logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) - vbd_ref = yield self._call_xenapi('VBD.create', vbd_rec) - logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, - vdi_ref) - defer.returnValue(vbd_ref) - - @defer.inlineCallbacks - def _create_vif(self, vm_ref, network_ref, mac_address): - """Create a VIF record. Returns a Deferred that gives the new - VIF reference.""" - - vif_rec = {} - vif_rec['device'] = '0' - vif_rec['network'] = network_ref - vif_rec['VM'] = vm_ref - vif_rec['MAC'] = mac_address - vif_rec['MTU'] = '1500' - vif_rec['other_config'] = {} - vif_rec['qos_algorithm_type'] = '' - vif_rec['qos_algorithm_params'] = {} - logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, - network_ref) - vif_ref = yield self._call_xenapi('VIF.create', vif_rec) - logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, - vm_ref, network_ref) - defer.returnValue(vif_ref) - - @defer.inlineCallbacks - def _find_network_with_bridge(self, bridge): - expr = 'field "bridge" = "%s"' % bridge - networks = yield self._call_xenapi('network.get_all_records_where', - expr) - if len(networks) == 1: - defer.returnValue(networks.keys()[0]) - elif len(networks) > 1: - raise Exception('Found non-unique network for bridge %s' % bridge) - else: - raise Exception('Found no network for bridge %s' % bridge) - - @defer.inlineCallbacks - def _fetch_image(self, image, user, project, use_sr): - """use_sr: True to put the image as a VDI in an SR, False to place - it on dom0's filesystem. The former is for VM disks, the latter for - its kernel and ramdisk (if external kernels are being used). - Returns a Deferred that gives the new VDI UUID.""" - - url = images.image_url(image) - access = AuthManager().get_access_key(user, project) - logging.debug("Asking xapi to fetch %s as %s" % (url, access)) - fn = use_sr and 'get_vdi' or 'get_kernel' - args = {} - args['src_url'] = url - args['username'] = access - args['password'] = user.secret - if use_sr: - args['add_partition'] = 'true' - task = yield self._async_call_plugin('objectstore', fn, args) - uuid = yield self._wait_for_task(task) - defer.returnValue(uuid) - - @defer.inlineCallbacks + self._vmops.spawn(instance) + def reboot(self, instance): - vm = yield self._lookup(instance.name) - if vm is None: - raise Exception('instance not present %s' % instance.name) - task = yield self._call_xenapi('Async.VM.clean_reboot', vm) - yield self._wait_for_task(task) + self._vmops.reboot(instance) - @defer.inlineCallbacks def destroy(self, instance): - vm = yield self._lookup(instance.name) - if vm is None: - # Don't complain, just return. This lets us clean up instances - # that have already disappeared from the underlying platform. - defer.returnValue(None) - # Get the VDIs related to the VM - vdis = yield self._lookup_vm_vdis(vm) - try: - task = yield self._call_xenapi('Async.VM.hard_shutdown', vm) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - # Disk clean-up - if vdis: - for vdi in vdis: - try: - task = yield self._call_xenapi('Async.VDI.destroy', vdi) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - try: - task = yield self._call_xenapi('Async.VM.destroy', vm) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - + self._vmops.destroy(instance) + def get_info(self, instance_id): - vm = self._lookup_blocking(instance_id) - if vm is None: - raise Exception('instance not present %s' % instance_id) - rec = self._conn.xenapi.VM.get_record(vm) - return {'state': XENAPI_POWER_STATE[rec['power_state']], - 'max_mem': long(rec['memory_static_max']) >> 10, - 'mem': long(rec['memory_dynamic_max']) >> 10, - 'num_cpu': rec['VCPUs_max'], - 'cpu_time': 0} - + return self._vmops.get_info(instance_id) + def get_console_output(self, instance): - return 'FAKE CONSOLE OUTPUT' + return self._vmops.get_console_output(instance) + + def attach_volume(self, instance_name, device_path, mountpoint): + return self._volumeops.attach_volume(instance_name, device_path, mountpoint) + + def detach_volume(self, instance_name, mountpoint): + return self._volumeops.detach_volume(instance_name, mountpoint) + + +class XenAPISession(object): + def __init__(self, url, user, pw): + self._session = XenAPI.Session(url) + self._session.login_with_password(user, pw) @utils.deferredToThread - def _lookup(self, i): - return self._lookup_blocking(i) - - def _lookup_blocking(self, i): - vms = self._conn.xenapi.VM.get_by_name_label(i) - n = len(vms) - if n == 0: - return None - elif n > 1: - raise Exception('duplicate name found: %s' % i) - else: - return vms[0] + def call_xenapi(self, method, *args): + """Call the specified XenAPI method on a background thread. Returns + a Deferred for the result.""" + f = self._session.xenapi + for m in method.split('.'): + f = f.__getattr__(m) + return f(*args) @utils.deferredToThread - def _lookup_vm_vdis(self, vm): - return self._lookup_vm_vdis_blocking(vm) + def async_call_plugin(self, plugin, fn, args): + """Call Async.host.call_plugin on a background thread. Returns a + Deferred with the task reference.""" + return _unwrap_plugin_exceptions( + self._session.xenapi.Async.host.call_plugin, + self._get_xenapi_host(), plugin, fn, args) - def _lookup_vm_vdis_blocking(self, vm): - # Firstly we get the VBDs, then the VDIs. - # TODO: do we leave the read-only devices? - vbds = self._conn.xenapi.VM.get_VBDs(vm) - vdis = [] - if vbds: - for vbd in vbds: - try: - vdi = self._conn.xenapi.VBD.get_VDI(vbd) - # Test valid VDI - record = self._conn.xenapi.VDI.get_record(vdi) - except Exception, exc: - logging.warn(exc) - else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis - else: - return None + def get_xenapi_host(self): + return self._session.xenapi.session.get_this_host(self._session.handle) - def _wait_for_task(self, task): + def wait_for_task(self, task): """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" d = defer.Deferred() reactor.callLater(0, self._poll_task, task, d) return d - + @utils.deferredToThread def _poll_task(self, task, deferred): """Poll the given XenAPI task, and fire the given Deferred if we get a result.""" try: #logging.debug('Polling task %s...', task) - status = self._conn.xenapi.task.get_status(task) + status = self._session.xenapi.task.get_status(task) if status == 'pending': reactor.callLater(FLAGS.xenapi_task_poll_interval, self._poll_task, task, deferred) elif status == 'success': - result = self._conn.xenapi.task.get_result(task) + result = self._session.xenapi.task.get_result(task) logging.info('Task %s status: success. %s', task, result) deferred.callback(_parse_xmlrpc_value(result)) else: - error_info = self._conn.xenapi.task.get_error_info(task) + error_info = self._session.xenapi.task.get_error_info(task) logging.warn('Task %s status: %s. %s', task, status, error_info) deferred.errback(XenAPI.Failure(error_info)) @@ -390,26 +187,6 @@ class XenAPIConnection(object): logging.warn(exc) deferred.errback(exc) - @utils.deferredToThread - def _call_xenapi(self, method, *args): - """Call the specified XenAPI method on a background thread. Returns - a Deferred for the result.""" - f = self._conn.xenapi - for m in method.split('.'): - f = f.__getattr__(m) - return f(*args) - - @utils.deferredToThread - def _async_call_plugin(self, plugin, fn, args): - """Call Async.host.call_plugin on a background thread. Returns a - Deferred with the task reference.""" - return _unwrap_plugin_exceptions( - self._conn.xenapi.Async.host.call_plugin, - self._get_xenapi_host(), plugin, fn, args) - - def _get_xenapi_host(self): - return self._conn.xenapi.session.get_this_host(self._conn.handle) - def _unwrap_plugin_exceptions(func, *args, **kwargs): try: diff --git a/nova/virt/xenapi/power_state.py b/nova/virt/xenapi/power_state.py new file mode 100644 index 0000000000..d2d8fba425 --- /dev/null +++ b/nova/virt/xenapi/power_state.py @@ -0,0 +1,26 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova.compute import power_state + + +XENAPI_POWER_STATE = { + 'Halted': power_state.SHUTDOWN, + 'Running': power_state.RUNNING, + 'Paused': power_state.PAUSED, + 'Suspended': power_state.SHUTDOWN, # FIXME + 'Crashed': power_state.CRASHED} + \ No newline at end of file diff --git a/nova/virt/xenapi/xenapi.py b/nova/virt/xenapi/xenapi.py new file mode 100644 index 0000000000..ddbef4303f --- /dev/null +++ b/nova/virt/xenapi/xenapi.py @@ -0,0 +1,439 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +A connection to XenServer or Xen Cloud Platform. + +The concurrency model for this class is as follows: + +All XenAPI calls are on a thread (using t.i.t.deferToThread, via the decorator +deferredToThread). They are remote calls, and so may hang for the usual +reasons. They should not be allowed to block the reactor thread. + +All long-running XenAPI calls (VM.start, VM.reboot, etc) are called async +(using XenAPI.VM.async_start etc). These return a task, which can then be +polled for completion. Polling is handled using reactor.callLater. + +This combination of techniques means that we don't block the reactor thread at +all, and at the same time we don't hold lots of threads waiting for +long-running operations. + +FIXME: get_info currently doesn't conform to these rules, and will block the +reactor thread if the VM.get_by_name_label or VM.get_record calls block. + +**Related Flags** + +:xenapi_connection_url: URL for connection to XenServer/Xen Cloud Platform. +:xenapi_connection_username: Username for connection to XenServer/Xen Cloud + Platform (default: root). +:xenapi_connection_password: Password for connection to XenServer/Xen Cloud + Platform. +:xenapi_task_poll_interval: The interval (seconds) used for polling of + remote tasks (Async.VM.start, etc) + (default: 0.5). + +""" + +import logging +import xmlrpclib + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import task + +from nova import db +from nova import flags +from nova import process +from nova import utils +from nova.auth.manager import AuthManager # wrap this one +from nova.compute import instance_types # wrap this one +from xenapi import power_state +from nova.virt import images # wrap this one + +XenAPI = None + + +FLAGS = flags.FLAGS +flags.DEFINE_string('xenapi_connection_url', + None, + 'URL for connection to XenServer/Xen Cloud Platform.' + ' Required if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_username', + 'root', + 'Username for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_password', + None, + 'Password for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_float('xenapi_task_poll_interval', + 0.5, + 'The interval used for polling of remote tasks ' + '(Async.VM.start, etc). Used only if ' + 'connection_type=xenapi.') + + +def get_connection(_): + """Note that XenAPI doesn't have a read-only connection mode, so + the read_only parameter is ignored.""" + # This is loaded late so that there's no need to install this + # library when not using XenAPI. + global XenAPI + if XenAPI is None: + XenAPI = __import__('XenAPI') + url = FLAGS.xenapi_connection_url + username = FLAGS.xenapi_connection_username + password = FLAGS.xenapi_connection_password + if not url or password is None: + raise Exception('Must specify xenapi_connection_url, ' + 'xenapi_connection_username (optionally), and ' + 'xenapi_connection_password to use ' + 'connection_type=xenapi') + return XenAPIConnection(url, username, password) + + +class XenAPISession(object): + def __init__(self, url, user, pw): + self._session = XenAPI.Session(url) + self._session.login_with_password(user, pw) + + def session(self): + return self._session + + def list_instances(self): + return [self._session.xenapi.VM.get_name_label(vm) \ + for vm in self._session.xenapi.VM.get_all()] + + @defer.inlineCallbacks + def spawn(self, instance): + vm = yield self._lookup(instance.name) + if vm is not None: + raise Exception('Attempted to create non-unique name %s' % + instance.name) + + network = db.project_get_network(None, instance.project_id) + network_ref = \ + yield self._find_network_with_bridge(network.bridge) + + user = AuthManager().get_user(instance.user_id) + project = AuthManager().get_project(instance.project_id) + vdi_uuid = yield self._fetch_image( + instance.image_id, user, project, True) + kernel = yield self._fetch_image( + instance.kernel_id, user, project, False) + ramdisk = yield self._fetch_image( + instance.ramdisk_id, user, project, False) + vdi_ref = yield self._call_xenapi('VDI.get_by_uuid', vdi_uuid) + + vm_ref = yield self._create_vm(instance, kernel, ramdisk) + yield self._create_vbd(vm_ref, vdi_ref, 0, True) + if network_ref: + yield self._create_vif(vm_ref, network_ref, instance.mac_address) + logging.debug('Starting VM %s...', vm_ref) + yield self._call_xenapi('VM.start', vm_ref, False, False) + logging.info('Spawning VM %s created %s.', instance.name, vm_ref) + + @defer.inlineCallbacks + def _create_vm(self, instance, kernel, ramdisk): + """Create a VM record. Returns a Deferred that gives the new + VM reference.""" + + instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] + mem = str(long(instance_type['memory_mb']) * 1024 * 1024) + vcpus = str(instance_type['vcpus']) + rec = { + 'name_label': instance.name, + 'name_description': '', + 'is_a_template': False, + 'memory_static_min': '0', + 'memory_static_max': mem, + 'memory_dynamic_min': mem, + 'memory_dynamic_max': mem, + 'VCPUs_at_startup': vcpus, + 'VCPUs_max': vcpus, + 'VCPUs_params': {}, + 'actions_after_shutdown': 'destroy', + 'actions_after_reboot': 'restart', + 'actions_after_crash': 'destroy', + 'PV_bootloader': '', + 'PV_kernel': kernel, + 'PV_ramdisk': ramdisk, + 'PV_args': 'root=/dev/xvda1', + 'PV_bootloader_args': '', + 'PV_legacy_args': '', + 'HVM_boot_policy': '', + 'HVM_boot_params': {}, + 'platform': {}, + 'PCI_bus': '', + 'recommendations': '', + 'affinity': '', + 'user_version': '0', + 'other_config': {}, + } + logging.debug('Created VM %s...', instance.name) + vm_ref = yield self._call_xenapi('VM.create', rec) + logging.debug('Created VM %s as %s.', instance.name, vm_ref) + defer.returnValue(vm_ref) + + @defer.inlineCallbacks + def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): + """Create a VBD record. Returns a Deferred that gives the new + VBD reference.""" + + vbd_rec = {} + vbd_rec['VM'] = vm_ref + vbd_rec['VDI'] = vdi_ref + vbd_rec['userdevice'] = str(userdevice) + vbd_rec['bootable'] = bootable + vbd_rec['mode'] = 'RW' + vbd_rec['type'] = 'disk' + vbd_rec['unpluggable'] = True + vbd_rec['empty'] = False + vbd_rec['other_config'] = {} + vbd_rec['qos_algorithm_type'] = '' + vbd_rec['qos_algorithm_params'] = {} + vbd_rec['qos_supported_algorithms'] = [] + logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) + vbd_ref = yield self._call_xenapi('VBD.create', vbd_rec) + logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, + vdi_ref) + defer.returnValue(vbd_ref) + + @defer.inlineCallbacks + def _create_vif(self, vm_ref, network_ref, mac_address): + """Create a VIF record. Returns a Deferred that gives the new + VIF reference.""" + + vif_rec = {} + vif_rec['device'] = '0' + vif_rec['network'] = network_ref + vif_rec['VM'] = vm_ref + vif_rec['MAC'] = mac_address + vif_rec['MTU'] = '1500' + vif_rec['other_config'] = {} + vif_rec['qos_algorithm_type'] = '' + vif_rec['qos_algorithm_params'] = {} + logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, + network_ref) + vif_ref = yield self._call_xenapi('VIF.create', vif_rec) + logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, + vm_ref, network_ref) + defer.returnValue(vif_ref) + + @defer.inlineCallbacks + def _find_network_with_bridge(self, bridge): + expr = 'field "bridge" = "%s"' % bridge + networks = yield self._call_xenapi('network.get_all_records_where', + expr) + if len(networks) == 1: + defer.returnValue(networks.keys()[0]) + elif len(networks) > 1: + raise Exception('Found non-unique network for bridge %s' % bridge) + else: + raise Exception('Found no network for bridge %s' % bridge) + + @defer.inlineCallbacks + def _fetch_image(self, image, user, project, use_sr): + """use_sr: True to put the image as a VDI in an SR, False to place + it on dom0's filesystem. The former is for VM disks, the latter for + its kernel and ramdisk (if external kernels are being used). + Returns a Deferred that gives the new VDI UUID.""" + + url = images.image_url(image) + access = AuthManager().get_access_key(user, project) + logging.debug("Asking xapi to fetch %s as %s" % (url, access)) + fn = use_sr and 'get_vdi' or 'get_kernel' + args = {} + args['src_url'] = url + args['username'] = access + args['password'] = user.secret + if use_sr: + args['add_partition'] = 'true' + task = yield self._async_call_plugin('objectstore', fn, args) + uuid = yield self._wait_for_task(task) + defer.returnValue(uuid) + + @defer.inlineCallbacks + def reboot(self, instance): + vm = yield self._lookup(instance.name) + if vm is None: + raise Exception('instance not present %s' % instance.name) + task = yield self._call_xenapi('Async.VM.clean_reboot', vm) + yield self._wait_for_task(task) + + @defer.inlineCallbacks + def destroy(self, instance): + vm = yield self._lookup(instance.name) + if vm is None: + # Don't complain, just return. This lets us clean up instances + # that have already disappeared from the underlying platform. + defer.returnValue(None) + # Get the VDIs related to the VM + vdis = yield self._lookup_vm_vdis(vm) + try: + task = yield self._call_xenapi('Async.VM.hard_shutdown', vm) + yield self._wait_for_task(task) + except Exception, exc: + logging.warn(exc) + # Disk clean-up + if vdis: + for vdi in vdis: + try: + task = yield self._call_xenapi('Async.VDI.destroy', vdi) + yield self._wait_for_task(task) + except Exception, exc: + logging.warn(exc) + try: + task = yield self._call_xenapi('Async.VM.destroy', vm) + yield self._wait_for_task(task) + except Exception, exc: + logging.warn(exc) + + def get_info(self, instance_id): + vm = self._lookup_blocking(instance_id) + if vm is None: + raise Exception('instance not present %s' % instance_id) + rec = self._session.xenapi.VM.get_record(vm) + return {'state': XENAPI_POWER_STATE[rec['power_state']], + 'max_mem': long(rec['memory_static_max']) >> 10, + 'mem': long(rec['memory_dynamic_max']) >> 10, + 'num_cpu': rec['VCPUs_max'], + 'cpu_time': 0} + + def get_console_output(self, instance): + return 'FAKE CONSOLE OUTPUT' + + @utils.deferredToThread + def _lookup(self, i): + return self._lookup_blocking(i) + + def _lookup_blocking(self, i): + vms = self._session.xenapi.VM.get_by_name_label(i) + n = len(vms) + if n == 0: + return None + elif n > 1: + raise Exception('duplicate name found: %s' % i) + else: + return vms[0] + + @utils.deferredToThread + def _lookup_vm_vdis(self, vm): + return self._lookup_vm_vdis_blocking(vm) + + def _lookup_vm_vdis_blocking(self, vm): + # Firstly we get the VBDs, then the VDIs. + # TODO: do we leave the read-only devices? + vbds = self._session.xenapi.VM.get_VBDs(vm) + vdis = [] + if vbds: + for vbd in vbds: + try: + vdi = self._session.xenapi.VBD.get_VDI(vbd) + # Test valid VDI + record = self._session.xenapi.VDI.get_record(vdi) + except Exception, exc: + logging.warn(exc) + else: + vdis.append(vdi) + if len(vdis) > 0: + return vdis + else: + return None + + def _wait_for_task(self, task): + """Return a Deferred that will give the result of the given task. + The task is polled until it completes.""" + d = defer.Deferred() + reactor.callLater(0, self._poll_task, task, d) + return d + + @utils.deferredToThread + def _poll_task(self, task, deferred): + """Poll the given XenAPI task, and fire the given Deferred if we + get a result.""" + try: + #logging.debug('Polling task %s...', task) + status = self._session.xenapi.task.get_status(task) + if status == 'pending': + reactor.callLater(FLAGS.xenapi_task_poll_interval, + self._poll_task, task, deferred) + elif status == 'success': + result = self._session.xenapi.task.get_result(task) + logging.info('Task %s status: success. %s', task, result) + deferred.callback(_parse_xmlrpc_value(result)) + else: + error_info = self._session.xenapi.task.get_error_info(task) + logging.warn('Task %s status: %s. %s', task, status, + error_info) + deferred.errback(XenAPI.Failure(error_info)) + #logging.debug('Polling task %s done.', task) + except Exception, exc: + logging.warn(exc) + deferred.errback(exc) + + @utils.deferredToThread + def _call_xenapi(self, method, *args): + """Call the specified XenAPI method on a background thread. Returns + a Deferred for the result.""" + f = self._session.xenapi + for m in method.split('.'): + f = f.__getattr__(m) + return f(*args) + + @utils.deferredToThread + def _async_call_plugin(self, plugin, fn, args): + """Call Async.host.call_plugin on a background thread. Returns a + Deferred with the task reference.""" + return _unwrap_plugin_exceptions( + self._session.xenapi.Async.host.call_plugin, + self._get_xenapi_host(), plugin, fn, args) + + def _get_xenapi_host(self): + return self._session.xenapi.session.get_this_host(self._session.handle) + + +def _unwrap_plugin_exceptions(func, *args, **kwargs): + try: + return func(*args, **kwargs) + except XenAPI.Failure, exc: + logging.debug("Got exception: %s", exc) + if (len(exc.details) == 4 and + exc.details[0] == 'XENAPI_PLUGIN_EXCEPTION' and + exc.details[2] == 'Failure'): + params = None + try: + params = eval(exc.details[3]) + except: + raise exc + raise XenAPI.Failure(params) + else: + raise + except xmlrpclib.ProtocolError, exc: + logging.debug("Got exception: %s", exc) + raise + + +def _parse_xmlrpc_value(val): + """Parse the given value as if it were an XML-RPC value. This is + sometimes used as the format for the task.result field.""" + if not val: + return val + x = xmlrpclib.loads( + '' + + val + + '') + return x[0][0] From 541f8ce212a33d14ac5ba48b3dde6c43a60bc368 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Sat, 27 Nov 2010 13:33:38 +0000 Subject: [PATCH 019/119] typos and pep8 fixes --- nova/virt/xenapi.py | 35 +++++++++++++++++++-------------- nova/virt/xenapi/power_state.py | 1 - 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 93c1192056..a17894c849 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -55,8 +55,8 @@ from twisted.internet import reactor from twisted.internet import task from xenapi import power_state -from xenapi import vmops -from xenapi import volumeops +from xenapi import VMOps +from xenapi import VolumeOps XenAPI = None @@ -102,30 +102,32 @@ def get_connection(_): class XenAPIConnection(object): def __init__(self, url, user, pw): - self._session = XenAPISession(url, user, pw) - self._vmops = VMOps(sef._session) - self._volumeops = volumeOps(self._session) - + session = XenAPISession(url, user, pw) + self._vmops = VMOps(session) + self._volumeops = VolumeOps(session) + def list_instances(self): return self._vmops.list_instances() - + def spawn(self, instance): self._vmops.spawn(instance) - + def reboot(self, instance): - self._vmops.reboot(instance) + self._vmops.reboot(instance) def destroy(self, instance): self._vmops.destroy(instance) - + def get_info(self, instance_id): return self._vmops.get_info(instance_id) - + def get_console_output(self, instance): - return self._vmops.get_console_output(instance) - + return self._vmops.get_console_output(instance) + def attach_volume(self, instance_name, device_path, mountpoint): - return self._volumeops.attach_volume(instance_name, device_path, mountpoint) + return self._volumeops.attach_volume(instance_name, + device_path, + mountpoint) def detach_volume(self, instance_name, mountpoint): return self._volumeops.detach_volume(instance_name, mountpoint) @@ -136,6 +138,9 @@ class XenAPISession(object): self._session = XenAPI.Session(url) self._session.login_with_password(user, pw) + def get_session(self): + return self._session + @utils.deferredToThread def call_xenapi(self, method, *args): """Call the specified XenAPI method on a background thread. Returns @@ -162,7 +167,7 @@ class XenAPISession(object): d = defer.Deferred() reactor.callLater(0, self._poll_task, task, d) return d - + @utils.deferredToThread def _poll_task(self, task, deferred): """Poll the given XenAPI task, and fire the given Deferred if we diff --git a/nova/virt/xenapi/power_state.py b/nova/virt/xenapi/power_state.py index d2d8fba425..5892f0f485 100644 --- a/nova/virt/xenapi/power_state.py +++ b/nova/virt/xenapi/power_state.py @@ -23,4 +23,3 @@ XENAPI_POWER_STATE = { 'Paused': power_state.PAUSED, 'Suspended': power_state.SHUTDOWN, # FIXME 'Crashed': power_state.CRASHED} - \ No newline at end of file From b6bed02342ac716b3cb3847fb54b5f285995f3b7 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Sun, 28 Nov 2010 01:49:28 +0000 Subject: [PATCH 020/119] further refactoring --- nova/virt/xenapi.py | 15 +- nova/virt/xenapi/network_utils.py | 52 ++++ nova/virt/xenapi/vm_utils.py | 206 ++++++++++++++ nova/virt/xenapi/vmops.py | 126 +++++++++ nova/virt/xenapi/xenapi.py | 439 ------------------------------ 5 files changed, 391 insertions(+), 447 deletions(-) create mode 100644 nova/virt/xenapi/network_utils.py create mode 100644 nova/virt/xenapi/vm_utils.py create mode 100644 nova/virt/xenapi/vmops.py delete mode 100644 nova/virt/xenapi/xenapi.py diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index a17894c849..2f2cef75ef 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -54,7 +54,6 @@ from twisted.internet import defer from twisted.internet import reactor from twisted.internet import task -from xenapi import power_state from xenapi import VMOps from xenapi import VolumeOps @@ -138,8 +137,11 @@ class XenAPISession(object): self._session = XenAPI.Session(url) self._session.login_with_password(user, pw) - def get_session(self): - return self._session + def get_xenapi(self): + return self._session.xenapi + + def get_xenapi_host(self): + return self._session.xenapi.session.get_this_host(self._session.handle) @utils.deferredToThread def call_xenapi(self, method, *args): @@ -149,17 +151,14 @@ class XenAPISession(object): for m in method.split('.'): f = f.__getattr__(m) return f(*args) - + @utils.deferredToThread def async_call_plugin(self, plugin, fn, args): """Call Async.host.call_plugin on a background thread. Returns a Deferred with the task reference.""" return _unwrap_plugin_exceptions( self._session.xenapi.Async.host.call_plugin, - self._get_xenapi_host(), plugin, fn, args) - - def get_xenapi_host(self): - return self._session.xenapi.session.get_this_host(self._session.handle) + self.get_xenapi_host(), plugin, fn, args) def wait_for_task(self, task): """Return a Deferred that will give the result of the given task. diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xenapi/network_utils.py new file mode 100644 index 0000000000..e062f916f0 --- /dev/null +++ b/nova/virt/xenapi/network_utils.py @@ -0,0 +1,52 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper methods for operations related to the management of network records and +their attributes like bridges, PIFs, QoS, as well as their lookup functions. +""" + +import logging +import xmlrpclib + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import task + +from nova import db +from nova import flags +from nova import process +from nova import utils +from nova.auth.manager import AuthManager # wrap this one +from nova.compute import instance_types # wrap this one +from nova.virt import images # wrap this one + +import power_state + + +class NetworkHelper(): + @classmethod + @defer.inlineCallbacks + def find_network_with_bridge(self, session, bridge): + expr = 'field "bridge" = "%s"' % bridge + networks = yield session.call_xenapi('network.get_all_records_where', + expr) + if len(networks) == 1: + defer.returnValue(networks.keys()[0]) + elif len(networks) > 1: + raise Exception('Found non-unique network for bridge %s' % bridge) + else: + raise Exception('Found no network for bridge %s' % bridge) \ No newline at end of file diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py new file mode 100644 index 0000000000..6fb409b26e --- /dev/null +++ b/nova/virt/xenapi/vm_utils.py @@ -0,0 +1,206 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper methods for operations related to the management of VM records and +their attributes like VDIs, VIFs, as well as their lookup functions. +""" + +import logging +import xmlrpclib + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import task + +from nova import db +from nova import flags +from nova import process +from nova import utils +from nova.auth.manager import AuthManager # wrap this one +from nova.compute import instance_types # wrap this one +from nova.virt import images # wrap this one + +import power_state + + +class VMHelper(): + @classmethod + @defer.inlineCallbacks + def create_vm(self, session, instance, kernel, ramdisk): + """Create a VM record. Returns a Deferred that gives the new + VM reference.""" + + instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] + mem = str(long(instance_type['memory_mb']) * 1024 * 1024) + vcpus = str(instance_type['vcpus']) + rec = { + 'name_label': instance.name, + 'name_description': '', + 'is_a_template': False, + 'memory_static_min': '0', + 'memory_static_max': mem, + 'memory_dynamic_min': mem, + 'memory_dynamic_max': mem, + 'VCPUs_at_startup': vcpus, + 'VCPUs_max': vcpus, + 'VCPUs_params': {}, + 'actions_after_shutdown': 'destroy', + 'actions_after_reboot': 'restart', + 'actions_after_crash': 'destroy', + 'PV_bootloader': '', + 'PV_kernel': kernel, + 'PV_ramdisk': ramdisk, + 'PV_args': 'root=/dev/xvda1', + 'PV_bootloader_args': '', + 'PV_legacy_args': '', + 'HVM_boot_policy': '', + 'HVM_boot_params': {}, + 'platform': {}, + 'PCI_bus': '', + 'recommendations': '', + 'affinity': '', + 'user_version': '0', + 'other_config': {}, + } + logging.debug('Created VM %s...', instance.name) + vm_ref = yield session.call_xenapi('VM.create', rec) + logging.debug('Created VM %s as %s.', instance.name, vm_ref) + defer.returnValue(vm_ref) + + @classmethod + @defer.inlineCallbacks + def create_vbd(self, session, vm_ref, vdi_ref, userdevice, bootable): + """Create a VBD record. Returns a Deferred that gives the new + VBD reference.""" + + vbd_rec = {} + vbd_rec['VM'] = vm_ref + vbd_rec['VDI'] = vdi_ref + vbd_rec['userdevice'] = str(userdevice) + vbd_rec['bootable'] = bootable + vbd_rec['mode'] = 'RW' + vbd_rec['type'] = 'disk' + vbd_rec['unpluggable'] = True + vbd_rec['empty'] = False + vbd_rec['other_config'] = {} + vbd_rec['qos_algorithm_type'] = '' + vbd_rec['qos_algorithm_params'] = {} + vbd_rec['qos_supported_algorithms'] = [] + logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) + vbd_ref = yield session.call_xenapi('VBD.create', vbd_rec) + logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, + vdi_ref) + defer.returnValue(vbd_ref) + + @classmethod + @defer.inlineCallbacks + def create_vif(self, session, vm_ref, network_ref, mac_address): + """Create a VIF record. Returns a Deferred that gives the new + VIF reference.""" + + vif_rec = {} + vif_rec['device'] = '0' + vif_rec['network'] = network_ref + vif_rec['VM'] = vm_ref + vif_rec['MAC'] = mac_address + vif_rec['MTU'] = '1500' + vif_rec['other_config'] = {} + vif_rec['qos_algorithm_type'] = '' + vif_rec['qos_algorithm_params'] = {} + logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, + network_ref) + vif_ref = yield session.call_xenapi('VIF.create', vif_rec) + logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, + vm_ref, network_ref) + defer.returnValue(vif_ref) + + @classmethod + @defer.inlineCallbacks + def find_network_with_bridge(self, session, bridge): + expr = 'field "bridge" = "%s"' % bridge + networks = yield session.call_xenapi('network.get_all_records_where', + expr) + if len(networks) == 1: + defer.returnValue(networks.keys()[0]) + elif len(networks) > 1: + raise Exception('Found non-unique network for bridge %s' % bridge) + else: + raise Exception('Found no network for bridge %s' % bridge) + + @classmethod + @defer.inlineCallbacks + def fetch_image(self, session, image, user, project, use_sr): + """use_sr: True to put the image as a VDI in an SR, False to place + it on dom0's filesystem. The former is for VM disks, the latter for + its kernel and ramdisk (if external kernels are being used). + Returns a Deferred that gives the new VDI UUID.""" + + url = images.image_url(image) + access = AuthManager().get_access_key(user, project) + logging.debug("Asking xapi to fetch %s as %s" % (url, access)) + fn = use_sr and 'get_vdi' or 'get_kernel' + args = {} + args['src_url'] = url + args['username'] = access + args['password'] = user.secret + if use_sr: + args['add_partition'] = 'true' + task = yield session.async_call_plugin('objectstore', fn, args) + uuid = yield session.wait_for_task(task) + defer.returnValue(uuid) + + @classmethod + @utils.deferredToThread + def lookup(self, session, i): + return VMHelper.lookup_blocking(i) + + @classmethod + def lookup_blocking(self, session, i): + vms = session.get_xenapi().VM.get_by_name_label(i) + n = len(vms) + if n == 0: + return None + elif n > 1: + raise Exception('duplicate name found: %s' % i) + else: + return vms[0] + + @classmethod + @utils.deferredToThread + def lookup_vm_vdis(self, session, vm): + return VMHelper.lookup_vm_vdis_blocking(session, vm) + + @classmethod + def lookup_vm_vdis_blocking(self, session, vm): + # Firstly we get the VBDs, then the VDIs. + # TODO: do we leave the read-only devices? + vbds = session.get_xenapi().VM.get_VBDs(vm) + vdis = [] + if vbds: + for vbd in vbds: + try: + vdi = session.get_xenapi().VBD.get_VDI(vbd) + # Test valid VDI + record = session.get_xenapi().VDI.get_record(vdi) + except Exception, exc: + logging.warn(exc) + else: + vdis.append(vdi) + if len(vdis) > 0: + return vdis + else: + return None \ No newline at end of file diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py new file mode 100644 index 0000000000..03b7fc614f --- /dev/null +++ b/nova/virt/xenapi/vmops.py @@ -0,0 +1,126 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Management class for VM-related functions (spawn, reboot, etc). +""" + +import logging +import xmlrpclib + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet import task + +from nova import db +from nova import flags +from nova import process +from nova import utils +from nova.auth.manager import AuthManager # wrap this one +from nova.compute import instance_types # wrap this one +from nova.virt import images # wrap this one + +import power_state +import VMHelper +import NetworkHelper + + +class VMOps(object): + def __init__(self, session): + self._session = session + + def list_instances(self): + return [self._session.get_xenapi().VM.get_name_label(vm) \ + for vm in self._session.get_xenapi().VM.get_all()] + + @defer.inlineCallbacks + def spawn(self, instance): + vm = yield VMHelper.lookup(self._session, instance.name) + if vm is not None: + raise Exception('Attempted to create non-unique name %s' % + instance.name) + + network = db.project_get_network(None, instance.project_id) + network_ref = \ + yield NetworkHelper.find_network_with_bridge(self._session, network.bridge) + + user = AuthManager().get_user(instance.user_id) + project = AuthManager().get_project(instance.project_id) + vdi_uuid = yield VMHelper.fetch_image(self._session, + instance.image_id, user, project, True) + kernel = yield VMHelper.fetch_image(self._session, + instance.kernel_id, user, project, False) + ramdisk = yield VMHelper.fetch_image(self._session, + instance.ramdisk_id, user, project, False) + vdi_ref = yield self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) + + vm_ref = yield VMHelper.create_vm(self._session, instance, kernel, ramdisk) + yield VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) + if network_ref: + yield VMHelper.create_vif(self._session, vm_ref, network_ref, instance.mac_address) + logging.debug('Starting VM %s...', vm_ref) + yield self._session.call_xenapi('VM.start', vm_ref, False, False) + logging.info('Spawning VM %s created %s.', instance.name, vm_ref) + + @defer.inlineCallbacks + def reboot(self, instance): + vm = yield VMHelper.lookup(self._session, instance.name) + if vm is None: + raise Exception('instance not present %s' % instance.name) + task = yield self._session.call_xenapi('Async.VM.clean_reboot', vm) + yield self._session.wait_for_task(task) + + @defer.inlineCallbacks + def destroy(self, instance): + vm = yield VMHelper.lookup(self._session, instance.name) + if vm is None: + # Don't complain, just return. This lets us clean up instances + # that have already disappeared from the underlying platform. + defer.returnValue(None) + # Get the VDIs related to the VM + vdis = yield VMHelper.lookup_vm_vdis(self._session, vm) + try: + task = yield self._session.call_xenapi('Async.VM.hard_shutdown', vm) + yield self._session.wait_for_task(task) + except Exception, exc: + logging.warn(exc) + # Disk clean-up + if vdis: + for vdi in vdis: + try: + task = yield self._session.call_xenapi('Async.VDI.destroy', vdi) + yield self._session.wait_for_task(task) + except Exception, exc: + logging.warn(exc) + try: + task = yield self._session.call_xenapi('Async.VM.destroy', vm) + yield self._session.wait_for_task(task) + except Exception, exc: + logging.warn(exc) + + def get_info(self, instance_id): + vm = VMHelper.lookup_blocking(self._session, instance_id) + if vm is None: + raise Exception('instance not present %s' % instance_id) + rec = self._session.get_xenapi().VM.get_record(vm) + return {'state': XENAPI_POWER_STATE[rec['power_state']], + 'max_mem': long(rec['memory_static_max']) >> 10, + 'mem': long(rec['memory_dynamic_max']) >> 10, + 'num_cpu': rec['VCPUs_max'], + 'cpu_time': 0} + + def get_console_output(self, instance): + return 'FAKE CONSOLE OUTPUT' \ No newline at end of file diff --git a/nova/virt/xenapi/xenapi.py b/nova/virt/xenapi/xenapi.py deleted file mode 100644 index ddbef4303f..0000000000 --- a/nova/virt/xenapi/xenapi.py +++ /dev/null @@ -1,439 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A connection to XenServer or Xen Cloud Platform. - -The concurrency model for this class is as follows: - -All XenAPI calls are on a thread (using t.i.t.deferToThread, via the decorator -deferredToThread). They are remote calls, and so may hang for the usual -reasons. They should not be allowed to block the reactor thread. - -All long-running XenAPI calls (VM.start, VM.reboot, etc) are called async -(using XenAPI.VM.async_start etc). These return a task, which can then be -polled for completion. Polling is handled using reactor.callLater. - -This combination of techniques means that we don't block the reactor thread at -all, and at the same time we don't hold lots of threads waiting for -long-running operations. - -FIXME: get_info currently doesn't conform to these rules, and will block the -reactor thread if the VM.get_by_name_label or VM.get_record calls block. - -**Related Flags** - -:xenapi_connection_url: URL for connection to XenServer/Xen Cloud Platform. -:xenapi_connection_username: Username for connection to XenServer/Xen Cloud - Platform (default: root). -:xenapi_connection_password: Password for connection to XenServer/Xen Cloud - Platform. -:xenapi_task_poll_interval: The interval (seconds) used for polling of - remote tasks (Async.VM.start, etc) - (default: 0.5). - -""" - -import logging -import xmlrpclib - -from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import task - -from nova import db -from nova import flags -from nova import process -from nova import utils -from nova.auth.manager import AuthManager # wrap this one -from nova.compute import instance_types # wrap this one -from xenapi import power_state -from nova.virt import images # wrap this one - -XenAPI = None - - -FLAGS = flags.FLAGS -flags.DEFINE_string('xenapi_connection_url', - None, - 'URL for connection to XenServer/Xen Cloud Platform.' - ' Required if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_username', - 'root', - 'Username for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_password', - None, - 'Password for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_float('xenapi_task_poll_interval', - 0.5, - 'The interval used for polling of remote tasks ' - '(Async.VM.start, etc). Used only if ' - 'connection_type=xenapi.') - - -def get_connection(_): - """Note that XenAPI doesn't have a read-only connection mode, so - the read_only parameter is ignored.""" - # This is loaded late so that there's no need to install this - # library when not using XenAPI. - global XenAPI - if XenAPI is None: - XenAPI = __import__('XenAPI') - url = FLAGS.xenapi_connection_url - username = FLAGS.xenapi_connection_username - password = FLAGS.xenapi_connection_password - if not url or password is None: - raise Exception('Must specify xenapi_connection_url, ' - 'xenapi_connection_username (optionally), and ' - 'xenapi_connection_password to use ' - 'connection_type=xenapi') - return XenAPIConnection(url, username, password) - - -class XenAPISession(object): - def __init__(self, url, user, pw): - self._session = XenAPI.Session(url) - self._session.login_with_password(user, pw) - - def session(self): - return self._session - - def list_instances(self): - return [self._session.xenapi.VM.get_name_label(vm) \ - for vm in self._session.xenapi.VM.get_all()] - - @defer.inlineCallbacks - def spawn(self, instance): - vm = yield self._lookup(instance.name) - if vm is not None: - raise Exception('Attempted to create non-unique name %s' % - instance.name) - - network = db.project_get_network(None, instance.project_id) - network_ref = \ - yield self._find_network_with_bridge(network.bridge) - - user = AuthManager().get_user(instance.user_id) - project = AuthManager().get_project(instance.project_id) - vdi_uuid = yield self._fetch_image( - instance.image_id, user, project, True) - kernel = yield self._fetch_image( - instance.kernel_id, user, project, False) - ramdisk = yield self._fetch_image( - instance.ramdisk_id, user, project, False) - vdi_ref = yield self._call_xenapi('VDI.get_by_uuid', vdi_uuid) - - vm_ref = yield self._create_vm(instance, kernel, ramdisk) - yield self._create_vbd(vm_ref, vdi_ref, 0, True) - if network_ref: - yield self._create_vif(vm_ref, network_ref, instance.mac_address) - logging.debug('Starting VM %s...', vm_ref) - yield self._call_xenapi('VM.start', vm_ref, False, False) - logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - - @defer.inlineCallbacks - def _create_vm(self, instance, kernel, ramdisk): - """Create a VM record. Returns a Deferred that gives the new - VM reference.""" - - instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] - mem = str(long(instance_type['memory_mb']) * 1024 * 1024) - vcpus = str(instance_type['vcpus']) - rec = { - 'name_label': instance.name, - 'name_description': '', - 'is_a_template': False, - 'memory_static_min': '0', - 'memory_static_max': mem, - 'memory_dynamic_min': mem, - 'memory_dynamic_max': mem, - 'VCPUs_at_startup': vcpus, - 'VCPUs_max': vcpus, - 'VCPUs_params': {}, - 'actions_after_shutdown': 'destroy', - 'actions_after_reboot': 'restart', - 'actions_after_crash': 'destroy', - 'PV_bootloader': '', - 'PV_kernel': kernel, - 'PV_ramdisk': ramdisk, - 'PV_args': 'root=/dev/xvda1', - 'PV_bootloader_args': '', - 'PV_legacy_args': '', - 'HVM_boot_policy': '', - 'HVM_boot_params': {}, - 'platform': {}, - 'PCI_bus': '', - 'recommendations': '', - 'affinity': '', - 'user_version': '0', - 'other_config': {}, - } - logging.debug('Created VM %s...', instance.name) - vm_ref = yield self._call_xenapi('VM.create', rec) - logging.debug('Created VM %s as %s.', instance.name, vm_ref) - defer.returnValue(vm_ref) - - @defer.inlineCallbacks - def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): - """Create a VBD record. Returns a Deferred that gives the new - VBD reference.""" - - vbd_rec = {} - vbd_rec['VM'] = vm_ref - vbd_rec['VDI'] = vdi_ref - vbd_rec['userdevice'] = str(userdevice) - vbd_rec['bootable'] = bootable - vbd_rec['mode'] = 'RW' - vbd_rec['type'] = 'disk' - vbd_rec['unpluggable'] = True - vbd_rec['empty'] = False - vbd_rec['other_config'] = {} - vbd_rec['qos_algorithm_type'] = '' - vbd_rec['qos_algorithm_params'] = {} - vbd_rec['qos_supported_algorithms'] = [] - logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) - vbd_ref = yield self._call_xenapi('VBD.create', vbd_rec) - logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, - vdi_ref) - defer.returnValue(vbd_ref) - - @defer.inlineCallbacks - def _create_vif(self, vm_ref, network_ref, mac_address): - """Create a VIF record. Returns a Deferred that gives the new - VIF reference.""" - - vif_rec = {} - vif_rec['device'] = '0' - vif_rec['network'] = network_ref - vif_rec['VM'] = vm_ref - vif_rec['MAC'] = mac_address - vif_rec['MTU'] = '1500' - vif_rec['other_config'] = {} - vif_rec['qos_algorithm_type'] = '' - vif_rec['qos_algorithm_params'] = {} - logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, - network_ref) - vif_ref = yield self._call_xenapi('VIF.create', vif_rec) - logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, - vm_ref, network_ref) - defer.returnValue(vif_ref) - - @defer.inlineCallbacks - def _find_network_with_bridge(self, bridge): - expr = 'field "bridge" = "%s"' % bridge - networks = yield self._call_xenapi('network.get_all_records_where', - expr) - if len(networks) == 1: - defer.returnValue(networks.keys()[0]) - elif len(networks) > 1: - raise Exception('Found non-unique network for bridge %s' % bridge) - else: - raise Exception('Found no network for bridge %s' % bridge) - - @defer.inlineCallbacks - def _fetch_image(self, image, user, project, use_sr): - """use_sr: True to put the image as a VDI in an SR, False to place - it on dom0's filesystem. The former is for VM disks, the latter for - its kernel and ramdisk (if external kernels are being used). - Returns a Deferred that gives the new VDI UUID.""" - - url = images.image_url(image) - access = AuthManager().get_access_key(user, project) - logging.debug("Asking xapi to fetch %s as %s" % (url, access)) - fn = use_sr and 'get_vdi' or 'get_kernel' - args = {} - args['src_url'] = url - args['username'] = access - args['password'] = user.secret - if use_sr: - args['add_partition'] = 'true' - task = yield self._async_call_plugin('objectstore', fn, args) - uuid = yield self._wait_for_task(task) - defer.returnValue(uuid) - - @defer.inlineCallbacks - def reboot(self, instance): - vm = yield self._lookup(instance.name) - if vm is None: - raise Exception('instance not present %s' % instance.name) - task = yield self._call_xenapi('Async.VM.clean_reboot', vm) - yield self._wait_for_task(task) - - @defer.inlineCallbacks - def destroy(self, instance): - vm = yield self._lookup(instance.name) - if vm is None: - # Don't complain, just return. This lets us clean up instances - # that have already disappeared from the underlying platform. - defer.returnValue(None) - # Get the VDIs related to the VM - vdis = yield self._lookup_vm_vdis(vm) - try: - task = yield self._call_xenapi('Async.VM.hard_shutdown', vm) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - # Disk clean-up - if vdis: - for vdi in vdis: - try: - task = yield self._call_xenapi('Async.VDI.destroy', vdi) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - try: - task = yield self._call_xenapi('Async.VM.destroy', vm) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - - def get_info(self, instance_id): - vm = self._lookup_blocking(instance_id) - if vm is None: - raise Exception('instance not present %s' % instance_id) - rec = self._session.xenapi.VM.get_record(vm) - return {'state': XENAPI_POWER_STATE[rec['power_state']], - 'max_mem': long(rec['memory_static_max']) >> 10, - 'mem': long(rec['memory_dynamic_max']) >> 10, - 'num_cpu': rec['VCPUs_max'], - 'cpu_time': 0} - - def get_console_output(self, instance): - return 'FAKE CONSOLE OUTPUT' - - @utils.deferredToThread - def _lookup(self, i): - return self._lookup_blocking(i) - - def _lookup_blocking(self, i): - vms = self._session.xenapi.VM.get_by_name_label(i) - n = len(vms) - if n == 0: - return None - elif n > 1: - raise Exception('duplicate name found: %s' % i) - else: - return vms[0] - - @utils.deferredToThread - def _lookup_vm_vdis(self, vm): - return self._lookup_vm_vdis_blocking(vm) - - def _lookup_vm_vdis_blocking(self, vm): - # Firstly we get the VBDs, then the VDIs. - # TODO: do we leave the read-only devices? - vbds = self._session.xenapi.VM.get_VBDs(vm) - vdis = [] - if vbds: - for vbd in vbds: - try: - vdi = self._session.xenapi.VBD.get_VDI(vbd) - # Test valid VDI - record = self._session.xenapi.VDI.get_record(vdi) - except Exception, exc: - logging.warn(exc) - else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis - else: - return None - - def _wait_for_task(self, task): - """Return a Deferred that will give the result of the given task. - The task is polled until it completes.""" - d = defer.Deferred() - reactor.callLater(0, self._poll_task, task, d) - return d - - @utils.deferredToThread - def _poll_task(self, task, deferred): - """Poll the given XenAPI task, and fire the given Deferred if we - get a result.""" - try: - #logging.debug('Polling task %s...', task) - status = self._session.xenapi.task.get_status(task) - if status == 'pending': - reactor.callLater(FLAGS.xenapi_task_poll_interval, - self._poll_task, task, deferred) - elif status == 'success': - result = self._session.xenapi.task.get_result(task) - logging.info('Task %s status: success. %s', task, result) - deferred.callback(_parse_xmlrpc_value(result)) - else: - error_info = self._session.xenapi.task.get_error_info(task) - logging.warn('Task %s status: %s. %s', task, status, - error_info) - deferred.errback(XenAPI.Failure(error_info)) - #logging.debug('Polling task %s done.', task) - except Exception, exc: - logging.warn(exc) - deferred.errback(exc) - - @utils.deferredToThread - def _call_xenapi(self, method, *args): - """Call the specified XenAPI method on a background thread. Returns - a Deferred for the result.""" - f = self._session.xenapi - for m in method.split('.'): - f = f.__getattr__(m) - return f(*args) - - @utils.deferredToThread - def _async_call_plugin(self, plugin, fn, args): - """Call Async.host.call_plugin on a background thread. Returns a - Deferred with the task reference.""" - return _unwrap_plugin_exceptions( - self._session.xenapi.Async.host.call_plugin, - self._get_xenapi_host(), plugin, fn, args) - - def _get_xenapi_host(self): - return self._session.xenapi.session.get_this_host(self._session.handle) - - -def _unwrap_plugin_exceptions(func, *args, **kwargs): - try: - return func(*args, **kwargs) - except XenAPI.Failure, exc: - logging.debug("Got exception: %s", exc) - if (len(exc.details) == 4 and - exc.details[0] == 'XENAPI_PLUGIN_EXCEPTION' and - exc.details[2] == 'Failure'): - params = None - try: - params = eval(exc.details[3]) - except: - raise exc - raise XenAPI.Failure(params) - else: - raise - except xmlrpclib.ProtocolError, exc: - logging.debug("Got exception: %s", exc) - raise - - -def _parse_xmlrpc_value(val): - """Parse the given value as if it were an XML-RPC value. This is - sometimes used as the format for the task.result field.""" - if not val: - return val - x = xmlrpclib.loads( - '' + - val + - '') - return x[0][0] From c10a6f3e97a5871ac0cdce97bde89b3cee59d336 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Sun, 28 Nov 2010 15:12:37 +0000 Subject: [PATCH 021/119] other round of refactoring --- nova/virt/xenapi/novadeps.py | 97 +++++++++++++++++++ nova/virt/xenapi/vm_utils.py | 32 ++---- nova/virt/xenapi/vmops.py | 41 ++++---- .../xenapi/{power_state.py => volumeops.py} | 22 +++-- 4 files changed, 141 insertions(+), 51 deletions(-) create mode 100644 nova/virt/xenapi/novadeps.py rename nova/virt/xenapi/{power_state.py => volumeops.py} (64%) diff --git a/nova/virt/xenapi/novadeps.py b/nova/virt/xenapi/novadeps.py new file mode 100644 index 0000000000..a4e5122630 --- /dev/null +++ b/nova/virt/xenapi/novadeps.py @@ -0,0 +1,97 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import db +from nova import flags +from nova import process +from nova import utils + +from nova.compute import power_state +from nova.auth.manager import AuthManager +from nova.compute import instance_types +from nova.virt import images + +XENAPI_POWER_STATE = { + 'Halted': power_state.SHUTDOWN, + 'Running': power_state.RUNNING, + 'Paused': power_state.PAUSED, + 'Suspended': power_state.SHUTDOWN, # FIXME + 'Crashed': power_state.CRASHED} + +class Instance(object): + + @classmethod + def get_name(self, instance): + return instance.name + + @classmethod + def get_type(self, instance): + return instance_types.INSTANCE_TYPES[instance.instance_type] + + @classmethod + def get_project(self, instance): + return AuthManager().get_project(instance.project_id) + + @classmethod + def get_project_id(self, instance): + return instance.project_id + + @classmethod + def get_image_id(self, instance): + return instance.image_id + + @classmethod + def get_kernel_id(self, instance): + return instance.kernel_id + + @classmethod + def get_ramdisk_id(self, instance): + return instance.ramdisk_id + + @classmethod + def get_network(self, instance): + return db.project_get_network(None, instance.project_id) + + @classmethod + def get_mac(self, instance): + return instance.mac_address + + @classmethod + def get_user(self, instance): + return AuthManager().get_user(instance.user_id) + + +class Network(object): + + @classmethod + def get_bridge(self, network): + return network.bridge + +class Image(object): + + @classmethod + def get_url(self, image): + return images.image_url(image) + +class User(object): + + @classmethod + def get_access(self, user, project): + return AuthManager().get_access_key(user, project) + + @classmethod + def get_secret(self, user): + return user.secret \ No newline at end of file diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 6fb409b26e..8329f0d7e2 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -30,11 +30,10 @@ from nova import db from nova import flags from nova import process from nova import utils -from nova.auth.manager import AuthManager # wrap this one -from nova.compute import instance_types # wrap this one -from nova.virt import images # wrap this one -import power_state +from novadeps import Instance +from novadeps import Image +from novadeps import User class VMHelper(): @@ -44,7 +43,7 @@ class VMHelper(): """Create a VM record. Returns a Deferred that gives the new VM reference.""" - instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] + instance_type = Instance.get_type(instance) mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) rec = { @@ -76,9 +75,9 @@ class VMHelper(): 'user_version': '0', 'other_config': {}, } - logging.debug('Created VM %s...', instance.name) + logging.debug('Created VM %s...', Instance.get_name(instance)) vm_ref = yield session.call_xenapi('VM.create', rec) - logging.debug('Created VM %s as %s.', instance.name, vm_ref) + logging.debug('Created VM %s as %s.', Instance.get_name(instance), vm_ref) defer.returnValue(vm_ref) @classmethod @@ -128,19 +127,6 @@ class VMHelper(): vm_ref, network_ref) defer.returnValue(vif_ref) - @classmethod - @defer.inlineCallbacks - def find_network_with_bridge(self, session, bridge): - expr = 'field "bridge" = "%s"' % bridge - networks = yield session.call_xenapi('network.get_all_records_where', - expr) - if len(networks) == 1: - defer.returnValue(networks.keys()[0]) - elif len(networks) > 1: - raise Exception('Found non-unique network for bridge %s' % bridge) - else: - raise Exception('Found no network for bridge %s' % bridge) - @classmethod @defer.inlineCallbacks def fetch_image(self, session, image, user, project, use_sr): @@ -149,14 +135,14 @@ class VMHelper(): its kernel and ramdisk (if external kernels are being used). Returns a Deferred that gives the new VDI UUID.""" - url = images.image_url(image) - access = AuthManager().get_access_key(user, project) + url = Image.get_url(image) + access = User.get_access(user, project) logging.debug("Asking xapi to fetch %s as %s" % (url, access)) fn = use_sr and 'get_vdi' or 'get_kernel' args = {} args['src_url'] = url args['username'] = access - args['password'] = user.secret + args['password'] = User.get_secret(user) if use_sr: args['add_partition'] = 'true' task = yield session.async_call_plugin('objectstore', fn, args) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 03b7fc614f..abb4225020 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -25,18 +25,14 @@ from twisted.internet import defer from twisted.internet import reactor from twisted.internet import task -from nova import db -from nova import flags -from nova import process -from nova import utils -from nova.auth.manager import AuthManager # wrap this one -from nova.compute import instance_types # wrap this one -from nova.virt import images # wrap this one -import power_state import VMHelper import NetworkHelper +from novadeps import XENAPI_POWER_STATE +from novadeps import Auth +from novadeps import Instance +from novadeps import Network class VMOps(object): def __init__(self, session): @@ -48,44 +44,45 @@ class VMOps(object): @defer.inlineCallbacks def spawn(self, instance): - vm = yield VMHelper.lookup(self._session, instance.name) + vm = yield VMHelper.lookup(self._session, Instance.get_name(instance)) if vm is not None: raise Exception('Attempted to create non-unique name %s' % - instance.name) + Instance.get_name(instance)) - network = db.project_get_network(None, instance.project_id) + network = Instance.get_network(instance) network_ref = \ - yield NetworkHelper.find_network_with_bridge(self._session, network.bridge) + yield NetworkHelper.find_network_with_bridge(self._session, Network.get_bridge(network)) - user = AuthManager().get_user(instance.user_id) - project = AuthManager().get_project(instance.project_id) + user = Instance.get_user(instance) + project = Instance.get_project(instance) vdi_uuid = yield VMHelper.fetch_image(self._session, - instance.image_id, user, project, True) + Instance.get_image_id(instance), user, project, True) kernel = yield VMHelper.fetch_image(self._session, - instance.kernel_id, user, project, False) + Instance.get_kernel_id(instance), user, project, False) ramdisk = yield VMHelper.fetch_image(self._session, - instance.ramdisk_id, user, project, False) + Instance.get_ramdisk_id(instance), user, project, False) vdi_ref = yield self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) vm_ref = yield VMHelper.create_vm(self._session, instance, kernel, ramdisk) yield VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) if network_ref: - yield VMHelper.create_vif(self._session, vm_ref, network_ref, instance.mac_address) + yield VMHelper.create_vif(self._session, vm_ref, network_ref, Instance.get_mac(instance)) logging.debug('Starting VM %s...', vm_ref) yield self._session.call_xenapi('VM.start', vm_ref, False, False) - logging.info('Spawning VM %s created %s.', instance.name, vm_ref) + logging.info('Spawning VM %s created %s.', Instance.get_name(instance), vm_ref) @defer.inlineCallbacks def reboot(self, instance): - vm = yield VMHelper.lookup(self._session, instance.name) + instance_name = Instance.get_name(instance) + vm = yield VMHelper.lookup(self._session, instance_name) if vm is None: - raise Exception('instance not present %s' % instance.name) + raise Exception('instance not present %s' % instance_name) task = yield self._session.call_xenapi('Async.VM.clean_reboot', vm) yield self._session.wait_for_task(task) @defer.inlineCallbacks def destroy(self, instance): - vm = yield VMHelper.lookup(self._session, instance.name) + vm = yield VMHelper.lookup(self._session, Instance.get_name(instance)) if vm is None: # Don't complain, just return. This lets us clean up instances # that have already disappeared from the underlying platform. diff --git a/nova/virt/xenapi/power_state.py b/nova/virt/xenapi/volumeops.py similarity index 64% rename from nova/virt/xenapi/power_state.py rename to nova/virt/xenapi/volumeops.py index 5892f0f485..f5b43adfb3 100644 --- a/nova/virt/xenapi/power_state.py +++ b/nova/virt/xenapi/volumeops.py @@ -14,12 +14,22 @@ # License for the specific language governing permissions and limitations # under the License. +""" +Management class for Storage-related functions (attach, detach, etc). +""" + +from twisted.internet import defer + +from nova import exception from nova.compute import power_state -XENAPI_POWER_STATE = { - 'Halted': power_state.SHUTDOWN, - 'Running': power_state.RUNNING, - 'Paused': power_state.PAUSED, - 'Suspended': power_state.SHUTDOWN, # FIXME - 'Crashed': power_state.CRASHED} +class VMOps(object): + def __init__(self, session): + self._session = session + + def attach_volume(self, instance_name, device_path, mountpoint): + return True + + def detach_volume(self, instance_name, mountpoint): + return True \ No newline at end of file From 9d26ad69bfeb88106a08f0f3f1e15ed621c18af2 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 29 Nov 2010 10:25:52 +0000 Subject: [PATCH 022/119] first cut of the refactoring of the XenAPIConnection class. Currently the class merged both the code for managing the XenAPI connection and the business logic for implementing Nova operations. If left like this, it would eventually become difficult to read, maintain and extend. The file was getting kind of big and cluttered, so a quick refactoring now will save a lot of headaches later --- nova/virt/xenapi/network_utils.py | 10 ---------- nova/virt/xenapi/vm_utils.py | 2 +- nova/virt/xenapi/vmops.py | 1 - nova/virt/xenapi/volumeops.py | 5 ----- 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xenapi/network_utils.py index e062f916f0..83ade13892 100644 --- a/nova/virt/xenapi/network_utils.py +++ b/nova/virt/xenapi/network_utils.py @@ -25,16 +25,6 @@ import xmlrpclib from twisted.internet import defer from twisted.internet import reactor from twisted.internet import task - -from nova import db -from nova import flags -from nova import process -from nova import utils -from nova.auth.manager import AuthManager # wrap this one -from nova.compute import instance_types # wrap this one -from nova.virt import images # wrap this one - -import power_state class NetworkHelper(): diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 8329f0d7e2..a1b444e417 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -152,7 +152,7 @@ class VMHelper(): @classmethod @utils.deferredToThread def lookup(self, session, i): - return VMHelper.lookup_blocking(i) + return VMHelper.lookup_blocking(session, i) @classmethod def lookup_blocking(self, session, i): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index abb4225020..c04a9f4ec4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -25,7 +25,6 @@ from twisted.internet import defer from twisted.internet import reactor from twisted.internet import task - import VMHelper import NetworkHelper diff --git a/nova/virt/xenapi/volumeops.py b/nova/virt/xenapi/volumeops.py index f5b43adfb3..fd316a0b8f 100644 --- a/nova/virt/xenapi/volumeops.py +++ b/nova/virt/xenapi/volumeops.py @@ -18,11 +18,6 @@ Management class for Storage-related functions (attach, detach, etc). """ -from twisted.internet import defer - -from nova import exception -from nova.compute import power_state - class VMOps(object): def __init__(self, session): From 9e34c9c7dc88d9e361c7f2d05e06b53ff68ee53f Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 29 Nov 2010 12:52:03 +0000 Subject: [PATCH 023/119] fixed deps --- nova/virt/xapi/__init__.py | 15 +++++++++ nova/virt/{xenapi => xapi}/network_utils.py | 16 ++++----- nova/virt/{xenapi => xapi}/novadeps.py | 31 ++++++++++-------- nova/virt/{xenapi => xapi}/vm_utils.py | 36 ++++++++++----------- nova/virt/{xenapi => xapi}/vmops.py | 35 ++++++++++---------- nova/virt/{xenapi => xapi}/volumeops.py | 4 +-- nova/virt/xenapi.py | 11 ++++--- 7 files changed, 83 insertions(+), 65 deletions(-) create mode 100644 nova/virt/xapi/__init__.py rename nova/virt/{xenapi => xapi}/network_utils.py (90%) rename nova/virt/{xenapi => xapi}/novadeps.py (93%) rename nova/virt/{xenapi => xapi}/vm_utils.py (92%) rename nova/virt/{xenapi => xapi}/vmops.py (87%) rename nova/virt/{xenapi => xapi}/volumeops.py (95%) diff --git a/nova/virt/xapi/__init__.py b/nova/virt/xapi/__init__.py new file mode 100644 index 0000000000..3d598c463c --- /dev/null +++ b/nova/virt/xapi/__init__.py @@ -0,0 +1,15 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xapi/network_utils.py similarity index 90% rename from nova/virt/xenapi/network_utils.py rename to nova/virt/xapi/network_utils.py index 83ade13892..b58b9159c2 100644 --- a/nova/virt/xenapi/network_utils.py +++ b/nova/virt/xapi/network_utils.py @@ -15,19 +15,17 @@ # under the License. """ -Helper methods for operations related to the management of network records and +Helper methods for operations related to the management of network records and their attributes like bridges, PIFs, QoS, as well as their lookup functions. """ -import logging -import xmlrpclib - from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import task - - + + class NetworkHelper(): + def __init__(self, session): + return + @classmethod @defer.inlineCallbacks def find_network_with_bridge(self, session, bridge): @@ -39,4 +37,4 @@ class NetworkHelper(): elif len(networks) > 1: raise Exception('Found non-unique network for bridge %s' % bridge) else: - raise Exception('Found no network for bridge %s' % bridge) \ No newline at end of file + raise Exception('Found no network for bridge %s' % bridge) diff --git a/nova/virt/xenapi/novadeps.py b/nova/virt/xapi/novadeps.py similarity index 93% rename from nova/virt/xenapi/novadeps.py rename to nova/virt/xapi/novadeps.py index a4e5122630..8cb5e3246f 100644 --- a/nova/virt/xenapi/novadeps.py +++ b/nova/virt/xapi/novadeps.py @@ -22,7 +22,7 @@ from nova import utils from nova.compute import power_state from nova.auth.manager import AuthManager from nova.compute import instance_types -from nova.virt import images +from nova.virt import images XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, @@ -30,33 +30,34 @@ XENAPI_POWER_STATE = { 'Paused': power_state.PAUSED, 'Suspended': power_state.SHUTDOWN, # FIXME 'Crashed': power_state.CRASHED} - + + class Instance(object): @classmethod def get_name(self, instance): return instance.name - + @classmethod def get_type(self, instance): return instance_types.INSTANCE_TYPES[instance.instance_type] - + @classmethod def get_project(self, instance): return AuthManager().get_project(instance.project_id) - + @classmethod def get_project_id(self, instance): return instance.project_id - + @classmethod def get_image_id(self, instance): return instance.image_id - + @classmethod def get_kernel_id(self, instance): return instance.kernel_id - + @classmethod def get_ramdisk_id(self, instance): return instance.ramdisk_id @@ -64,28 +65,30 @@ class Instance(object): @classmethod def get_network(self, instance): return db.project_get_network(None, instance.project_id) - + @classmethod def get_mac(self, instance): return instance.mac_address - + @classmethod def get_user(self, instance): return AuthManager().get_user(instance.user_id) - + class Network(object): @classmethod def get_bridge(self, network): return network.bridge - + + class Image(object): @classmethod def get_url(self, image): return images.image_url(image) - + + class User(object): @classmethod @@ -94,4 +97,4 @@ class User(object): @classmethod def get_secret(self, user): - return user.secret \ No newline at end of file + return user.secret diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xapi/vm_utils.py similarity index 92% rename from nova/virt/xenapi/vm_utils.py rename to nova/virt/xapi/vm_utils.py index a1b444e417..41f687ccbb 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xapi/vm_utils.py @@ -15,35 +15,32 @@ # under the License. """ -Helper methods for operations related to the management of VM records and +Helper methods for operations related to the management of VM records and their attributes like VDIs, VIFs, as well as their lookup functions. """ import logging -import xmlrpclib from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import task -from nova import db -from nova import flags -from nova import process from nova import utils from novadeps import Instance from novadeps import Image from novadeps import User - - + + class VMHelper(): + def __init__(self, session): + return + @classmethod @defer.inlineCallbacks def create_vm(self, session, instance, kernel, ramdisk): """Create a VM record. Returns a Deferred that gives the new VM reference.""" - instance_type = Instance.get_type(instance) + instance_type = Instance.get_type(instance) mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) rec = { @@ -77,10 +74,11 @@ class VMHelper(): } logging.debug('Created VM %s...', Instance.get_name(instance)) vm_ref = yield session.call_xenapi('VM.create', rec) - logging.debug('Created VM %s as %s.', Instance.get_name(instance), vm_ref) + logging.debug('Created VM %s as %s.', + Instance.get_name(instance), vm_ref) defer.returnValue(vm_ref) - - @classmethod + + @classmethod @defer.inlineCallbacks def create_vbd(self, session, vm_ref, vdi_ref, userdevice, bootable): """Create a VBD record. Returns a Deferred that gives the new @@ -105,7 +103,7 @@ class VMHelper(): vdi_ref) defer.returnValue(vbd_ref) - @classmethod + @classmethod @defer.inlineCallbacks def create_vif(self, session, vm_ref, network_ref, mac_address): """Create a VIF record. Returns a Deferred that gives the new @@ -147,9 +145,9 @@ class VMHelper(): args['add_partition'] = 'true' task = yield session.async_call_plugin('objectstore', fn, args) uuid = yield session.wait_for_task(task) - defer.returnValue(uuid) - - @classmethod + defer.returnValue(uuid) + + @classmethod @utils.deferredToThread def lookup(self, session, i): return VMHelper.lookup_blocking(session, i) @@ -165,7 +163,7 @@ class VMHelper(): else: return vms[0] - @classmethod + @classmethod @utils.deferredToThread def lookup_vm_vdis(self, session, vm): return VMHelper.lookup_vm_vdis_blocking(session, vm) @@ -189,4 +187,4 @@ class VMHelper(): if len(vdis) > 0: return vdis else: - return None \ No newline at end of file + return None diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xapi/vmops.py similarity index 87% rename from nova/virt/xenapi/vmops.py rename to nova/virt/xapi/vmops.py index c04a9f4ec4..d6ea5e7db7 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xapi/vmops.py @@ -19,24 +19,21 @@ Management class for VM-related functions (spawn, reboot, etc). """ import logging -import xmlrpclib from twisted.internet import defer -from twisted.internet import reactor -from twisted.internet import task - -import VMHelper -import NetworkHelper from novadeps import XENAPI_POWER_STATE -from novadeps import Auth from novadeps import Instance from novadeps import Network +from vm_utils import VMHelper +from network_utils import NetworkHelper + + class VMOps(object): def __init__(self, session): self._session = session - + def list_instances(self): return [self._session.get_xenapi().VM.get_name_label(vm) \ for vm in self._session.get_xenapi().VM.get_all()] @@ -48,9 +45,9 @@ class VMOps(object): raise Exception('Attempted to create non-unique name %s' % Instance.get_name(instance)) - network = Instance.get_network(instance) + bridge = Network.get_bridge(Instance.get_network(instance)) network_ref = \ - yield NetworkHelper.find_network_with_bridge(self._session, Network.get_bridge(network)) + yield NetworkHelper.find_network_with_bridge(self._session, bridge) user = Instance.get_user(instance) project = Instance.get_project(instance) @@ -61,14 +58,16 @@ class VMOps(object): ramdisk = yield VMHelper.fetch_image(self._session, Instance.get_ramdisk_id(instance), user, project, False) vdi_ref = yield self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - - vm_ref = yield VMHelper.create_vm(self._session, instance, kernel, ramdisk) + vm_ref = yield VMHelper.create_vm(self._session, + instance, kernel, ramdisk) yield VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) if network_ref: - yield VMHelper.create_vif(self._session, vm_ref, network_ref, Instance.get_mac(instance)) + yield VMHelper.create_vif(self._session, vm_ref, + network_ref, Instance.get_mac(instance)) logging.debug('Starting VM %s...', vm_ref) yield self._session.call_xenapi('VM.start', vm_ref, False, False) - logging.info('Spawning VM %s created %s.', Instance.get_name(instance), vm_ref) + logging.info('Spawning VM %s created %s.', Instance.get_name(instance), + vm_ref) @defer.inlineCallbacks def reboot(self, instance): @@ -89,7 +88,8 @@ class VMOps(object): # Get the VDIs related to the VM vdis = yield VMHelper.lookup_vm_vdis(self._session, vm) try: - task = yield self._session.call_xenapi('Async.VM.hard_shutdown', vm) + task = yield self._session.call_xenapi('Async.VM.hard_shutdown', + vm) yield self._session.wait_for_task(task) except Exception, exc: logging.warn(exc) @@ -97,7 +97,8 @@ class VMOps(object): if vdis: for vdi in vdis: try: - task = yield self._session.call_xenapi('Async.VDI.destroy', vdi) + task = yield self._session.call_xenapi('Async.VDI.destroy', + vdi) yield self._session.wait_for_task(task) except Exception, exc: logging.warn(exc) @@ -119,4 +120,4 @@ class VMOps(object): 'cpu_time': 0} def get_console_output(self, instance): - return 'FAKE CONSOLE OUTPUT' \ No newline at end of file + return 'FAKE CONSOLE OUTPUT' diff --git a/nova/virt/xenapi/volumeops.py b/nova/virt/xapi/volumeops.py similarity index 95% rename from nova/virt/xenapi/volumeops.py rename to nova/virt/xapi/volumeops.py index fd316a0b8f..23f79adf71 100644 --- a/nova/virt/xenapi/volumeops.py +++ b/nova/virt/xapi/volumeops.py @@ -19,7 +19,7 @@ Management class for Storage-related functions (attach, detach, etc). """ -class VMOps(object): +class VolumeOps(object): def __init__(self, session): self._session = session @@ -27,4 +27,4 @@ class VMOps(object): return True def detach_volume(self, instance_name, mountpoint): - return True \ No newline at end of file + return True diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 2f2cef75ef..613f19f823 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -52,10 +52,13 @@ import xmlrpclib from twisted.internet import defer from twisted.internet import reactor -from twisted.internet import task +#from twisted.internet import task -from xenapi import VMOps -from xenapi import VolumeOps +from nova import flags +from nova import utils + +from xapi.vmops import VMOps +from xapi.volumeops import VolumeOps XenAPI = None @@ -151,7 +154,7 @@ class XenAPISession(object): for m in method.split('.'): f = f.__getattr__(m) return f(*args) - + @utils.deferredToThread def async_call_plugin(self, plugin, fn, args): """Call Async.host.call_plugin on a background thread. Returns a From a7fe9d8cbc4bb6b2ca4306c0adff46edcb2fce6f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 29 Nov 2010 14:02:03 +0100 Subject: [PATCH 024/119] Add include_package_data=True to setup.py. This makes sure the various templates get installed into the python path when running "python setup.py install". --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ec0014478b..d88bc1e6f8 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup(name='nova', cmdclass={ 'sdist': local_sdist, 'build_sphinx' : local_BuildDoc }, packages=find_packages(exclude=['bin', 'smoketests']), + include_package_data=True, scripts=['bin/nova-api', 'bin/nova-compute', 'bin/nova-dhcpbridge', From a82581cbada92d0e274438757f7beb3ed335da1b Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 29 Nov 2010 16:31:31 +0000 Subject: [PATCH 025/119] pep8 fixes and further round of refactoring --- nova/virt/connection.py | 4 ++-- nova/virt/{xapi => xenapi}/__init__.py | 0 nova/virt/{xapi => xenapi}/network_utils.py | 0 nova/virt/{xapi => xenapi}/novadeps.py | 5 ++++- nova/virt/{xapi => xenapi}/vm_utils.py | 2 +- nova/virt/{xapi => xenapi}/vmops.py | 0 nova/virt/{xapi => xenapi}/volumeops.py | 0 nova/virt/{xenapi.py => xenapi_conn.py} | 5 ++--- 8 files changed, 9 insertions(+), 7 deletions(-) rename nova/virt/{xapi => xenapi}/__init__.py (100%) rename nova/virt/{xapi => xenapi}/network_utils.py (100%) rename nova/virt/{xapi => xenapi}/novadeps.py (92%) rename nova/virt/{xapi => xenapi}/vm_utils.py (99%) rename nova/virt/{xapi => xenapi}/vmops.py (100%) rename nova/virt/{xapi => xenapi}/volumeops.py (100%) rename nova/virt/{xenapi.py => xenapi_conn.py} (98%) diff --git a/nova/virt/connection.py b/nova/virt/connection.py index 11f0fa8ced..c40bb4bb4d 100644 --- a/nova/virt/connection.py +++ b/nova/virt/connection.py @@ -25,7 +25,7 @@ import sys from nova import flags from nova.virt import fake from nova.virt import libvirt_conn -from nova.virt import xenapi +from nova.virt import xenapi_conn FLAGS = flags.FLAGS @@ -61,7 +61,7 @@ def get_connection(read_only=False): elif t == 'libvirt': conn = libvirt_conn.get_connection(read_only) elif t == 'xenapi': - conn = xenapi.get_connection(read_only) + conn = xenapi_conn.get_connection(read_only) else: raise Exception('Unknown connection type "%s"' % t) diff --git a/nova/virt/xapi/__init__.py b/nova/virt/xenapi/__init__.py similarity index 100% rename from nova/virt/xapi/__init__.py rename to nova/virt/xenapi/__init__.py diff --git a/nova/virt/xapi/network_utils.py b/nova/virt/xenapi/network_utils.py similarity index 100% rename from nova/virt/xapi/network_utils.py rename to nova/virt/xenapi/network_utils.py diff --git a/nova/virt/xapi/novadeps.py b/nova/virt/xenapi/novadeps.py similarity index 92% rename from nova/virt/xapi/novadeps.py rename to nova/virt/xenapi/novadeps.py index 8cb5e3246f..ba62468fbc 100644 --- a/nova/virt/xapi/novadeps.py +++ b/nova/virt/xenapi/novadeps.py @@ -18,6 +18,7 @@ from nova import db from nova import flags from nova import process from nova import utils +from nova import context from nova.compute import power_state from nova.auth.manager import AuthManager @@ -64,7 +65,9 @@ class Instance(object): @classmethod def get_network(self, instance): - return db.project_get_network(None, instance.project_id) + # TODO: is ge_admin_context the right context to retrieve? + return db.project_get_network(context.get_admin_context(), + instance.project_id) @classmethod def get_mac(self, instance): diff --git a/nova/virt/xapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py similarity index 99% rename from nova/virt/xapi/vm_utils.py rename to nova/virt/xenapi/vm_utils.py index 41f687ccbb..b68df27919 100644 --- a/nova/virt/xapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -33,7 +33,7 @@ from novadeps import User class VMHelper(): def __init__(self, session): return - + @classmethod @defer.inlineCallbacks def create_vm(self, session, instance, kernel, ramdisk): diff --git a/nova/virt/xapi/vmops.py b/nova/virt/xenapi/vmops.py similarity index 100% rename from nova/virt/xapi/vmops.py rename to nova/virt/xenapi/vmops.py diff --git a/nova/virt/xapi/volumeops.py b/nova/virt/xenapi/volumeops.py similarity index 100% rename from nova/virt/xapi/volumeops.py rename to nova/virt/xenapi/volumeops.py diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi_conn.py similarity index 98% rename from nova/virt/xenapi.py rename to nova/virt/xenapi_conn.py index 613f19f823..0a73b47741 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi_conn.py @@ -52,13 +52,12 @@ import xmlrpclib from twisted.internet import defer from twisted.internet import reactor -#from twisted.internet import task from nova import flags from nova import utils -from xapi.vmops import VMOps -from xapi.volumeops import VolumeOps +from xenapi.vmops import VMOps +from xenapi.volumeops import VolumeOps XenAPI = None From 2cc492240fab447a62e7ca3ea1c16744baad9256 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 29 Nov 2010 21:23:21 +0000 Subject: [PATCH 026/119] update of nova.sh because default flagfile moved --- contrib/nova.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 6033c9e3a3..7407422b3d 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -23,7 +23,7 @@ MYSQL_PASS=${MYSQL_PASS:-nova} TEST=${TEST:-0} USE_LDAP=${USE_LDAP:-0} LIBVIRT_TYPE=${LIBVIRT_TYPE:-qemu} -NET_MAN=${NET_MAN:-FlatDHCPManager} +NET_MAN=${NET_MAN:-VlanManager} # NOTE(vish): If you are using FlatDHCP on multiple hosts, set the interface # below but make sure that the interface doesn't already have an # ip or you risk breaking things. @@ -42,10 +42,10 @@ else fi mkdir -p /etc/nova -cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF +cat >$NOVA_DIR/bin/nova.conf << NOVA_CONF_EOF --verbose --nodaemon ---dhcpbridge_flagfile=/etc/nova/nova-manage.conf +--dhcpbridge_flagfile=$NOVA_DIR/bin/nova.conf --FAKE_subdomain=ec2 --network_manager=nova.network.manager.$NET_MAN --cc_host=$HOST_IP @@ -56,7 +56,7 @@ cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF NOVA_CONF_EOF if [ -n "$FLAT_INTERFACE" ]; then - echo "--flat_interface=$FLAT_INTERFACE" >>/etc/nova/nova-manage.conf + echo "--flat_interface=$FLAT_INTERFACE" >>$NOVA_DIR/bin/nova.conf fi if [ "$CMD" == "branch" ]; then @@ -142,12 +142,12 @@ if [ "$CMD" == "run" ]; then # nova api crashes if we start it with a regular screen command, # so send the start command by forcing text into the window. - screen_it api "$NOVA_DIR/bin/nova-api --flagfile=/etc/nova/nova-manage.conf" - screen_it objectstore "$NOVA_DIR/bin/nova-objectstore --flagfile=/etc/nova/nova-manage.conf" - screen_it compute "$NOVA_DIR/bin/nova-compute --flagfile=/etc/nova/nova-manage.conf" - screen_it network "$NOVA_DIR/bin/nova-network --flagfile=/etc/nova/nova-manage.conf" - screen_it scheduler "$NOVA_DIR/bin/nova-scheduler --flagfile=/etc/nova/nova-manage.conf" - screen_it volume "$NOVA_DIR/bin/nova-volume --flagfile=/etc/nova/nova-manage.conf" + screen_it api "$NOVA_DIR/bin/nova-api" + screen_it objectstore "$NOVA_DIR/bin/nova-objectstore" + screen_it compute "$NOVA_DIR/bin/nova-compute" + screen_it network "$NOVA_DIR/bin/nova-network" + screen_it scheduler "$NOVA_DIR/bin/nova-scheduler" + screen_it volume "$NOVA_DIR/bin/nova-volume" screen_it test ". $NOVA_DIR/novarc" screen -S nova -x fi From e6dde30724ac47f6abeb5eaa56a68fb9ac166397 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 29 Nov 2010 23:04:54 +0100 Subject: [PATCH 027/119] Correctly handle imageId list passed to DescribeImages API call. --- nova/api/ec2/cloud.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 9327bf0d44..9cabd2e7d3 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -994,7 +994,10 @@ class CloudController(object): return True def describe_images(self, context, image_id=None, **kwargs): - imageSet = self.image_service.index(context, image_id) + # Note: image_id is a list! + imageSet = self.image_service.index(context) + if image_id: + imageSet = filter(lambda x: x['imageId'] in image_id, imageSet) return {'imagesSet': imageSet} def deregister_image(self, context, image_id, **kwargs): From 8ee658e7f6da2484377bec7652f37df7259f9e8a Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 29 Nov 2010 17:26:05 -0600 Subject: [PATCH 028/119] Return the correct server_management_url --- nova/api/openstack/auth.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index ff428ff701..f91742b378 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -47,7 +47,7 @@ class BasicApiAuthManager(object): except KeyError: return faults.Fault(webob.exc.HTTPUnauthorized()) - token, user = self._authorize_user(username, key) + token, user = self._authorize_user(username, key, req) if user and token: res = webob.Response() res.headers['X-Auth-Token'] = token.token_hash @@ -82,8 +82,13 @@ class BasicApiAuthManager(object): return {'id': user.id} return None - def _authorize_user(self, username, key): - """ Generates a new token and assigns it to a user """ + def _authorize_user(self, username, key, req): + """Generates a new token and assigns it to a user. + + username - string + key - string API key + req - webob.Request object + """ user = self.auth.get_user_from_access_key(key) if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, @@ -91,12 +96,10 @@ class BasicApiAuthManager(object): token_dict = {} token_dict['token_hash'] = token_hash token_dict['cdn_management_url'] = '' - token_dict['server_management_url'] = self._get_server_mgmt_url() + # Same as auth url, e.g. http://foo.org:8774/baz/v1.0 + token_dict['server_management_url'] = req.url token_dict['storage_url'] = '' token_dict['user_id'] = user.id token = self.db.auth_create_token(self.context, token_dict) return token, user return None, None - - def _get_server_mgmt_url(self): - return 'https://%s/v1.0/' % self.host From e82afc902020d7bf7bc60141a629287599d8796a Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 29 Nov 2010 17:38:51 -0600 Subject: [PATCH 029/119] remove FAKE_subdomain reference --- contrib/nova.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 6033c9e3a3..7f111d2b34 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -46,7 +46,6 @@ cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF --verbose --nodaemon --dhcpbridge_flagfile=/etc/nova/nova-manage.conf ---FAKE_subdomain=ec2 --network_manager=nova.network.manager.$NET_MAN --cc_host=$HOST_IP --routing_source_ip=$HOST_IP From 6d097a220846c54cb11b4a0e480f282e50db6058 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 30 Nov 2010 09:19:32 +0100 Subject: [PATCH 030/119] Rename imageSet variable to images. --- nova/api/ec2/cloud.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 9cabd2e7d3..884372ce7b 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -995,10 +995,10 @@ class CloudController(object): def describe_images(self, context, image_id=None, **kwargs): # Note: image_id is a list! - imageSet = self.image_service.index(context) + images = self.image_service.index(context) if image_id: - imageSet = filter(lambda x: x['imageId'] in image_id, imageSet) - return {'imagesSet': imageSet} + images = filter(lambda x: x['imageId'] in image_id, images) + return {'imagesSet': images} def deregister_image(self, context, image_id, **kwargs): self.image_service.deregister(context, image_id) From 41b3faf113d7591e61b03678dc13cd9ef031efbb Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 30 Nov 2010 10:40:17 -0500 Subject: [PATCH 031/119] If only I weren't so lazy. --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 29f4b88747..4b75995dce 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -69,7 +69,7 @@ class Test(unittest.TestCase): self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], - "https://foo/v1.0/") + "http://foo/v1.0/") self.assertEqual(result.headers['X-CDN-Management-Url'], "") self.assertEqual(result.headers['X-Storage-Url'], "") From 84fdd48fe2db20661f076884810f0c726630452f Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 30 Nov 2010 13:52:46 -0500 Subject: [PATCH 032/119] Fix unit tests --- nova/api/openstack/auth.py | 5 +---- nova/tests/api/openstack/fakes.py | 1 - nova/tests/api/openstack/test_auth.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index f91742b378..2050359154 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -23,10 +23,7 @@ class Context(object): class BasicApiAuthManager(object): """ Implements a somewhat rudimentary version of OpenStack Auth""" - def __init__(self, host=None, db_driver=None): - if not host: - host = FLAGS.host - self.host = host + def __init__(self, db_driver=None): if not db_driver: db_driver = FLAGS.db_driver self.db = utils.import_object(db_driver) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 639a2ebe4e..6e91ca7bbf 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -54,7 +54,6 @@ def fake_auth_init(self): self.db = FakeAuthDatabase() self.context = Context() self.auth = FakeAuthManager() - self.host = 'foo' @webob.dec.wsgify diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 4b75995dce..14e720be41 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -62,7 +62,7 @@ class Test(unittest.TestCase): f = fakes.FakeAuthManager() f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) - req = webob.Request.blank('/v1.0/') + req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' result = req.get_response(nova.api.API('os')) From aaee43a74264d5e6a4ccf638f882b19d477c3c9f Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 30 Nov 2010 23:12:19 +0000 Subject: [PATCH 033/119] Added a script to use OpenDJ as an LDAP server instead of OpenLDAP. Also modified nova.sh to add an USE_OPENDJ option, that will be checked when USE_LDAP is set. --- contrib/nova.sh | 10 +++- nova/auth/opendj.sh | 119 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 nova/auth/opendj.sh diff --git a/contrib/nova.sh b/contrib/nova.sh index 1a9f93a3b1..7eb934eca9 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -22,6 +22,8 @@ USE_MYSQL=${USE_MYSQL:-0} MYSQL_PASS=${MYSQL_PASS:-nova} TEST=${TEST:-0} USE_LDAP=${USE_LDAP:-0} +# Use OpenDJ instead of OpenLDAP when using LDAP +USE_OPENDJ=${USE_OPENDJ:-0} LIBVIRT_TYPE=${LIBVIRT_TYPE:-qemu} NET_MAN=${NET_MAN:-VlanManager} # NOTE(vish): If you are using FlatDHCP on multiple hosts, set the interface @@ -113,7 +115,13 @@ if [ "$CMD" == "run" ]; then rm $NOVA_DIR/nova.sqlite fi if [ "$USE_LDAP" == 1 ]; then - sudo $NOVA_DIR/nova/auth/slap.sh + if [ "$USE_OPENDJ" == 1 ]; then + echo '--ldap_user_dn=cn=Directory Manager' >> \ + /etc/nova/nova-manage.conf + sudo $NOVA_DIR/nova/auth/opendj.sh + else + sudo $NOVA_DIR/nova/auth/slap.sh + fi fi rm -rf $NOVA_DIR/instances mkdir -p $NOVA_DIR/instances diff --git a/nova/auth/opendj.sh b/nova/auth/opendj.sh new file mode 100755 index 0000000000..8052c077d0 --- /dev/null +++ b/nova/auth/opendj.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# LDAP INSTALL SCRIPT - IS IDEMPOTENT, does not scrub users + +apt-get install -y ldap-utils python-ldap openjdk-6-jre + +if [ ! -d "/usr/opendj" ] +then + # TODO(rlane): Wikimedia Foundation is the current package maintainer. + # After the package is included in Ubuntu's channel, change this. + wget http://apt.wikimedia.org/wikimedia/pool/main/o/opendj/opendj_2.4.0-7_amd64.deb + dpkg -i opendj_2.4.0-7_amd64.deb +fi + +abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` +schemapath='/var/opendj/instance/config/schema' +cp $abspath/openssh-lpk_sun.schema $schemapath/97-openssh-lpk_sun.ldif +cp $abspath/nova_sun.schema $schemapath/98-nova_sun.ldif +chown opendj:opendj $schemapath/97-openssh-lpk_sun.ldif +chown opendj:opendj $schemapath/98-nova_sun.ldif + +cat >/etc/ldap/ldap.conf </etc/ldap/base.ldif < Date: Wed, 1 Dec 2010 11:50:25 +0100 Subject: [PATCH 034/119] Move cc_host and cc_port flags into nova/network/linux_net.py. They weren't used anywhere else. Make cc_host default to nova.utils.get_my_ip() instead of 127.0.0.1. cc_host is used to set up forwarding to the meta-data service, and the kernel doesn't allow routing to a loopback device, so 127.0.0.1 is a poor default. --- nova/flags.py | 2 -- nova/network/linux_net.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index cb9fa105b4..1f94feb08b 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -223,8 +223,6 @@ DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('cc_host', '127.0.0.1', 'ip of api server') -DEFINE_integer('cc_port', 8773, 'cloud controller port') DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', 'Url to ec2 api server') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 7b00e65d42..0fefd94156 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -46,6 +46,8 @@ flags.DEFINE_string('vlan_interface', 'eth0', 'network device for vlans') flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') +flags.DEFINE_string('cc_host', utils.get_my_ip(), 'ip of api server') +flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') flags.DEFINE_string('routing_source_ip', '127.0.0.1', 'Public IP of network host') flags.DEFINE_bool('use_nova_chains', False, From 11dddd7ca4f4264ef3a8f1e251601c1d8fd7a626 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 10:44:51 -0600 Subject: [PATCH 035/119] Move default_flagfile() call to where it will be parsed in time to load the flagfile --- bin/nova-compute | 3 ++- bin/nova-instancemonitor | 2 +- bin/nova-network | 2 +- bin/nova-objectstore | 2 +- bin/nova-scheduler | 2 +- bin/nova-volume | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index a66477af53..1054852c4c 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -38,8 +38,9 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() # pylint: disable=C0103 + diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index a7b7fb0c68..9b6c40e820 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -42,10 +42,10 @@ logging.getLogger('boto').setLevel(logging.WARN) if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() logging.warn('Starting instance monitor') # pylint: disable-msg=C0103 monitor = monitor.InstanceMonitor() diff --git a/bin/nova-network b/bin/nova-network index 342a63058c..d1fb552612 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -38,8 +38,8 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() # pylint: disable-msg=C0103 diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 728f2ee5bc..00ae27af93 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -42,8 +42,8 @@ FLAGS = flags.FLAGS if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = handler.get_application() # pylint: disable-msg=C0103 diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 069b5a6fa9..4d1a40cf10 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -38,8 +38,8 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() diff --git a/bin/nova-volume b/bin/nova-volume index 26148b0ecc..e7281d6c0b 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -38,8 +38,8 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() # pylint: disable-msg=C0103 From 6956057ac490c788cb94fbfd0af7fe6e91a7ca96 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 1 Dec 2010 09:24:39 -0800 Subject: [PATCH 036/119] Broke parts of compute manager out into compute.api to separate what gets run on the API side vs the worker side. --- nova/api/ec2/cloud.py | 15 +-- nova/api/openstack/servers.py | 5 +- nova/compute/api.py | 207 +++++++++++++++++++++++++++++++++ nova/compute/manager.py | 169 --------------------------- nova/db/base.py | 36 ++++++ nova/manager.py | 10 +- nova/tests/compute_unittest.py | 8 +- 7 files changed, 262 insertions(+), 188 deletions(-) create mode 100644 nova/compute/api.py create mode 100644 nova/db/base.py diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index c694579674..6c09175007 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -39,6 +39,7 @@ from nova import flags from nova import quota from nova import rpc from nova import utils +from nova.compute import api as compute_api from nova.compute import instance_types from nova.api import cloud from nova.image.s3 import S3ImageService @@ -94,7 +95,7 @@ class CloudController(object): """ def __init__(self): self.network_manager = utils.import_object(FLAGS.network_manager) - self.compute_manager = utils.import_object(FLAGS.compute_manager) + self.compute_api = compute_api.ComputeAPI() self.image_service = S3ImageService() self.setup() @@ -255,7 +256,7 @@ class CloudController(object): return True def describe_security_groups(self, context, group_name=None, **kwargs): - self.compute_manager.ensure_default_security_group(context) + self.compute_api.ensure_default_security_group(context) if context.user.is_admin(): groups = db.security_group_get_all(context) else: @@ -353,7 +354,7 @@ class CloudController(object): return False def revoke_security_group_ingress(self, context, group_name, **kwargs): - self.compute_manager.ensure_default_security_group(context) + self.compute_api.ensure_default_security_group(context) security_group = db.security_group_get_by_name(context, context.project_id, group_name) @@ -378,7 +379,7 @@ class CloudController(object): # for these operations, so support for newer API versions # is sketchy. def authorize_security_group_ingress(self, context, group_name, **kwargs): - self.compute_manager.ensure_default_security_group(context) + self.compute_api.ensure_default_security_group(context) security_group = db.security_group_get_by_name(context, context.project_id, group_name) @@ -414,7 +415,7 @@ class CloudController(object): return source_project_id def create_security_group(self, context, group_name, group_description): - self.compute_manager.ensure_default_security_group(context) + self.compute_api.ensure_default_security_group(context) if db.security_group_exists(context, context.project_id, group_name): raise exception.ApiError('group %s already exists' % group_name) @@ -748,7 +749,7 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) - instances = self.compute_manager.create_instances(context, + instances = self.compute_api.create_instances(context, instance_types.get_by_type(kwargs.get('instance_type', None)), self.image_service, kwargs['image_id'], @@ -789,7 +790,7 @@ class CloudController(object): id_str) continue now = datetime.datetime.utcnow() - self.compute_manager.update_instance(context, + self.compute_api.update_instance(context, instance_ref['id'], state_description='terminating', state=0, diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index e1e2bf7fd4..8242c5b443 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -27,6 +27,7 @@ from nova import wsgi from nova import context from nova.api import cloud from nova.api.openstack import faults +from nova.compute import api as compute_api from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack @@ -95,7 +96,7 @@ class Controller(wsgi.Controller): db_driver = FLAGS.db_driver self.db_driver = utils.import_object(db_driver) self.network_manager = utils.import_object(FLAGS.network_manager) - self.compute_manager = utils.import_object(FLAGS.compute_manager) + self.compute_api = compute_api.ComputeAPI() super(Controller, self).__init__() def index(self, req): @@ -147,7 +148,7 @@ class Controller(wsgi.Controller): user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) key_pair = self.db_driver.key_pair_get_all_by_user(None, user_id)[0] - instances = self.compute_manager.create_instances(ctxt, + instances = self.compute_api.create_instances(ctxt, instance_types.get_by_flavor_id(env['server']['flavorId']), utils.import_object(FLAGS.image_service), env['server']['imageId'], diff --git a/nova/compute/api.py b/nova/compute/api.py new file mode 100644 index 0000000000..e678be85d5 --- /dev/null +++ b/nova/compute/api.py @@ -0,0 +1,207 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Handles all API requests relating to instances (guest vms). +""" + +import logging +import time + +from nova import db +from nova import exception +from nova import flags +from nova import quota +from nova import rpc +from nova import utils +from nova.compute import instance_types +from nova.db import base + +FLAGS = flags.FLAGS + + +def generate_default_hostname(internal_id): + """Default function to generate a hostname given an instance reference.""" + return str(internal_id) + + +class ComputeAPI(base.Base): + """API for interacting with the compute manager.""" + + def __init__(self, **kwargs): + self.network_manager = utils.import_object(FLAGS.network_manager) + super(ComputeAPI, self).__init__(**kwargs) + + # TODO(eday): network_topic arg should go away once we push network + # allocation into the scheduler or compute worker. + def create_instances(self, context, instance_type, image_service, image_id, + network_topic, min_count=1, max_count=1, + kernel_id=None, ramdisk_id=None, name='', + description='', user_data='', key_name=None, + key_data=None, security_group='default', + generate_hostname=generate_default_hostname): + """Create the number of instances requested if quote and + other arguments check out ok.""" + + num_instances = quota.allowed_instances(context, max_count, + instance_type) + if num_instances < min_count: + logging.warn("Quota exceeeded for %s, tried to run %s instances", + context.project_id, min_count) + raise quota.QuotaError("Instance quota exceeded. You can only " + "run %s more instances of this type." % + num_instances, "InstanceLimitExceeded") + + is_vpn = image_id == FLAGS.vpn_image_id + if not is_vpn: + image = image_service.show(context, image_id) + if kernel_id is None: + kernel_id = image.get('kernelId', FLAGS.default_kernel) + if ramdisk_id is None: + ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) + + # Make sure we have access to kernel and ramdisk + image_service.show(context, kernel_id) + image_service.show(context, ramdisk_id) + + if security_group is None: + security_group = ['default'] + if not type(security_group) is list: + security_group = [security_group] + + security_groups = [] + self.ensure_default_security_group(context) + for security_group_name in security_group: + group = db.security_group_get_by_name(context, + context.project_id, + security_group_name) + security_groups.append(group['id']) + + if key_data is None and key_name: + key_pair = db.key_pair_get(context, context.user_id, key_name) + key_data = key_pair['public_key'] + + type_data = instance_types.INSTANCE_TYPES[instance_type] + base_options = { + 'reservation_id': utils.generate_uid('r'), + 'server_name': name, + 'image_id': image_id, + 'kernel_id': kernel_id, + 'ramdisk_id': ramdisk_id, + 'state_description': 'scheduling', + 'user_id': context.user_id, + 'project_id': context.project_id, + 'launch_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'instance_type': instance_type, + 'memory_mb': type_data['memory_mb'], + 'vcpus': type_data['vcpus'], + 'local_gb': type_data['local_gb'], + 'display_name': name, + 'display_description': description, + 'key_name': key_name, + 'key_data': key_data} + + elevated = context.elevated() + instances = [] + logging.debug("Going to run %s instances...", num_instances) + for num in range(num_instances): + instance = dict(mac_address=utils.generate_mac(), + launch_index=num, + **base_options) + instance_ref = self.create_instance(context, security_groups, + **instance) + instance_id = instance_ref['id'] + internal_id = instance_ref['internal_id'] + hostname = generate_hostname(internal_id) + self.update_instance(context, instance_id, hostname=hostname) + instances.append(dict(id=instance_id, internal_id=internal_id, + hostname=hostname, **instance)) + + # TODO(vish): This probably should be done in the scheduler + # or in compute as a call. The network should be + # allocated after the host is assigned and setup + # can happen at the same time. + address = self.network_manager.allocate_fixed_ip(context, + instance_id, + is_vpn) + rpc.cast(elevated, + network_topic, + {"method": "setup_fixed_ip", + "args": {"address": address}}) + + logging.debug("Casting to scheduler for %s/%s's instance %s" % + (context.project_id, context.user_id, instance_id)) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "run_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id}}) + + return instances + + def ensure_default_security_group(self, context): + try: + db.security_group_get_by_name(context, context.project_id, + 'default') + except exception.NotFound: + values = {'name': 'default', + 'description': 'default', + 'user_id': context.user_id, + 'project_id': context.project_id} + group = db.security_group_create(context, values) + + def create_instance(self, context, security_groups=None, **kwargs): + """Creates the instance in the datastore and returns the + new instance as a mapping + + :param context: The security context + :param security_groups: list of security group ids to + attach to the instance + :param kwargs: All additional keyword args are treated + as data fields of the instance to be + created + + :retval Returns a mapping of the instance information + that has just been created + + """ + instance_ref = self.db.instance_create(context, kwargs) + inst_id = instance_ref['id'] + + elevated = context.elevated() + if not security_groups: + security_groups = [] + for security_group_id in security_groups: + self.db.instance_add_security_group(elevated, + inst_id, + security_group_id) + return instance_ref + + def update_instance(self, context, instance_id, **kwargs): + """Updates the instance in the datastore. + + :param context: The security context + :param instance_id: ID of the instance to update + :param kwargs: All additional keyword args are treated + as data fields of the instance to be + updated + + :retval None + + """ + self.db.instance_update(context, instance_id, kwargs) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3f870f8666..a25b8f6f36 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -36,7 +36,6 @@ termination. import datetime import logging -import time from twisted.internet import defer @@ -44,13 +43,9 @@ from nova import db from nova import exception from nova import flags from nova import manager -from nova import quota -from nova import rpc from nova import utils -from nova.compute import instance_types from nova.compute import power_state - FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', utils.abspath('../instances'), 'where instances are stored on disk') @@ -58,11 +53,6 @@ flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', 'Driver to use for volume creation') -def generate_default_hostname(internal_id): - """Default function to generate a hostname given an instance reference.""" - return str(internal_id) - - class ComputeManager(manager.Manager): """Manages the running instances from creation to destruction.""" @@ -94,165 +84,6 @@ class ComputeManager(manager.Manager): """This call passes stright through to the virtualization driver.""" yield self.driver.refresh_security_group(security_group_id) - # TODO(eday): network_topic arg should go away once we push network - # allocation into the scheduler or compute worker. - def create_instances(self, context, instance_type, image_service, image_id, - network_topic, min_count=1, max_count=1, - kernel_id=None, ramdisk_id=None, name='', - description='', user_data='', key_name=None, - key_data=None, security_group='default', - generate_hostname=generate_default_hostname): - """Create the number of instances requested if quote and - other arguments check out ok.""" - - num_instances = quota.allowed_instances(context, max_count, - instance_type) - if num_instances < min_count: - logging.warn("Quota exceeeded for %s, tried to run %s instances", - context.project_id, min_count) - raise quota.QuotaError("Instance quota exceeded. You can only " - "run %s more instances of this type." % - num_instances, "InstanceLimitExceeded") - - is_vpn = image_id == FLAGS.vpn_image_id - if not is_vpn: - image = image_service.show(context, image_id) - if kernel_id is None: - kernel_id = image.get('kernelId', FLAGS.default_kernel) - if ramdisk_id is None: - ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) - - # Make sure we have access to kernel and ramdisk - image_service.show(context, kernel_id) - image_service.show(context, ramdisk_id) - - if security_group is None: - security_group = ['default'] - if not type(security_group) is list: - security_group = [security_group] - - security_groups = [] - self.ensure_default_security_group(context) - for security_group_name in security_group: - group = db.security_group_get_by_name(context, - context.project_id, - security_group_name) - security_groups.append(group['id']) - - if key_data is None and key_name: - key_pair = db.key_pair_get(context, context.user_id, key_name) - key_data = key_pair['public_key'] - - type_data = instance_types.INSTANCE_TYPES[instance_type] - base_options = { - 'reservation_id': utils.generate_uid('r'), - 'server_name': name, - 'image_id': image_id, - 'kernel_id': kernel_id, - 'ramdisk_id': ramdisk_id, - 'state_description': 'scheduling', - 'user_id': context.user_id, - 'project_id': context.project_id, - 'launch_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), - 'instance_type': instance_type, - 'memory_mb': type_data['memory_mb'], - 'vcpus': type_data['vcpus'], - 'local_gb': type_data['local_gb'], - 'display_name': name, - 'display_description': description, - 'key_name': key_name, - 'key_data': key_data} - - elevated = context.elevated() - instances = [] - logging.debug("Going to run %s instances...", num_instances) - for num in range(num_instances): - instance = dict(mac_address=utils.generate_mac(), - launch_index=num, - **base_options) - instance_ref = self.create_instance(context, security_groups, - **instance) - instance_id = instance_ref['id'] - internal_id = instance_ref['internal_id'] - hostname = generate_hostname(internal_id) - self.update_instance(context, instance_id, hostname=hostname) - instances.append(dict(id=instance_id, internal_id=internal_id, - hostname=hostname, **instance)) - - # TODO(vish): This probably should be done in the scheduler - # or in compute as a call. The network should be - # allocated after the host is assigned and setup - # can happen at the same time. - address = self.network_manager.allocate_fixed_ip(context, - instance_id, - is_vpn) - rpc.cast(elevated, - network_topic, - {"method": "setup_fixed_ip", - "args": {"address": address}}) - - logging.debug("Casting to scheduler for %s/%s's instance %s" % - (context.project_id, context.user_id, instance_id)) - rpc.cast(context, - FLAGS.scheduler_topic, - {"method": "run_instance", - "args": {"topic": FLAGS.compute_topic, - "instance_id": instance_id}}) - - return instances - - def ensure_default_security_group(self, context): - try: - db.security_group_get_by_name(context, context.project_id, - 'default') - except exception.NotFound: - values = {'name': 'default', - 'description': 'default', - 'user_id': context.user_id, - 'project_id': context.project_id} - group = db.security_group_create(context, values) - - def create_instance(self, context, security_groups=None, **kwargs): - """Creates the instance in the datastore and returns the - new instance as a mapping - - :param context: The security context - :param security_groups: list of security group ids to - attach to the instance - :param kwargs: All additional keyword args are treated - as data fields of the instance to be - created - - :retval Returns a mapping of the instance information - that has just been created - - """ - instance_ref = self.db.instance_create(context, kwargs) - inst_id = instance_ref['id'] - - elevated = context.elevated() - if not security_groups: - security_groups = [] - for security_group_id in security_groups: - self.db.instance_add_security_group(elevated, - inst_id, - security_group_id) - return instance_ref - - def update_instance(self, context, instance_id, **kwargs): - """Updates the instance in the datastore. - - :param context: The security context - :param instance_id: ID of the instance to update - :param kwargs: All additional keyword args are treated - as data fields of the instance to be - updated - - :retval None - - """ - self.db.instance_update(context, instance_id, kwargs) - @defer.inlineCallbacks @exception.wrap_exception def run_instance(self, context, instance_id, **_kwargs): diff --git a/nova/db/base.py b/nova/db/base.py new file mode 100644 index 0000000000..1d1e80866b --- /dev/null +++ b/nova/db/base.py @@ -0,0 +1,36 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Base class for classes that need modular database access. +""" + +from nova import utils +from nova import flags + +FLAGS = flags.FLAGS +flags.DEFINE_string('db_driver', 'nova.db.api', + 'driver to use for database access') + + +class Base(object): + """DB driver is injected in the init method""" + def __init__(self, db_driver=None): + if not db_driver: + db_driver = FLAGS.db_driver + self.db = utils.import_object(db_driver) # pylint: disable-msg=C0103 diff --git a/nova/manager.py b/nova/manager.py index a6efb8732c..5b61f7a4cb 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -53,23 +53,19 @@ This module provides Manager, a base class for managers. from nova import utils from nova import flags +from nova.db import base from twisted.internet import defer FLAGS = flags.FLAGS -flags.DEFINE_string('db_driver', 'nova.db.api', - 'driver to use for volume creation') -class Manager(object): - """DB driver is injected in the init method""" +class Manager(base.Base): def __init__(self, host=None, db_driver=None): if not host: host = FLAGS.host self.host = host - if not db_driver: - db_driver = FLAGS.db_driver - self.db = utils.import_object(db_driver) # pylint: disable-msg=C0103 + super(Manager, self).__init__(db_driver) @defer.inlineCallbacks def periodic_tasks(self, context=None): diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 71a1a4457d..8f6f35b35e 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -31,6 +31,7 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager +from nova.compute import api as compute_api FLAGS = flags.FLAGS @@ -43,6 +44,7 @@ class ComputeTestCase(test.TrialTestCase): self.flags(connection_type='fake', network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) + self.compute_api = compute_api.ComputeAPI() self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') @@ -76,9 +78,9 @@ class ComputeTestCase(test.TrialTestCase): 'user_id': self.user.id, 'project_id': self.project.id} group = db.security_group_create(self.context, values) - ref = self.compute.create_instance(self.context, - security_groups=[group['id']], - **inst) + ref = self.compute_api.create_instance(self.context, + security_groups=[group['id']], + **inst) # reload to get groups instance_ref = db.instance_get(self.context, ref['id']) try: From 6e6b5325bfc5233b243ed9f8279694136d605ddf Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 11:43:20 -0600 Subject: [PATCH 037/119] Removed a blank line. --- bin/nova-compute | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/nova-compute b/bin/nova-compute index 1054852c4c..ac6378f754 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -43,4 +43,3 @@ if __name__ == '__main__': if __name__ == '__builtin__': application = service.Service.create() # pylint: disable=C0103 - From 93c7bbf98f0396718724cbf1d4d2f3953078776c Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 14:18:24 -0600 Subject: [PATCH 038/119] Remove duplicate field and make OpenStack API return server.name for EC2-API-created instances --- nova/api/openstack/servers.py | 5 ++--- nova/db/sqlalchemy/models.py | 3 +-- nova/tests/api/openstack/test_servers.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 1d8aa2fa4d..44e69b82cf 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,7 +63,7 @@ def _entity_detail(inst): inst_dict = {} mapped_keys = dict(status='state', imageId='image_id', - flavorId='instance_type', name='server_name', id='id') + flavorId='instance_type', name='display_name', id='id') for k, v in mapped_keys.iteritems(): inst_dict[k] = inst[v] @@ -78,7 +78,7 @@ def _entity_detail(inst): def _entity_inst(inst): """ Filters all model attributes save for id and name """ - return dict(server=dict(id=inst['id'], name=inst['server_name'])) + return dict(server=dict(id=inst['id'], name=inst['display_name'])) class Controller(wsgi.Controller): @@ -213,7 +213,6 @@ class Controller(wsgi.Controller): if not image: raise Exception("Image not found") - inst['server_name'] = env['server']['name'] inst['image_id'] = image_id inst['user_id'] = user_id inst['launch_time'] = ltime diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 01b5cf3507..fe0a9a9216 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -178,8 +178,6 @@ class Instance(BASE, NovaBase): kernel_id = Column(String(255)) ramdisk_id = Column(String(255)) - server_name = Column(String(255)) - # image_id = Column(Integer, ForeignKey('images.id'), nullable=True) # kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) # ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) @@ -212,6 +210,7 @@ class Instance(BASE, NovaBase): launched_at = Column(DateTime) terminated_at = Column(DateTime) + # User editable field for display in user-facing UIs display_name = Column(String(255)) display_description = Column(String(255)) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 8cfc6c45a3..530d067606 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -44,7 +44,7 @@ def return_servers(context, user_id=1): def stub_instance(id, user_id=1): - return Instance(id=id, state=0, image_id=10, server_name='server%s' % id, + return Instance(id=id, state=0, image_id=10, display_name='server%s' % id, user_id=user_id) From fdf0aa30a1127eb8311a599dfdad9653ac699154 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 14:55:42 -0600 Subject: [PATCH 039/119] Todd points out that the API doesn't require a display_name, so let's make a default. That way the OpenStack API can rest assured that its server responses will always have a name key. --- nova/compute/manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 50a9d316b4..0893db9fc5 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -99,6 +99,8 @@ class ComputeManager(manager.Manager): that has just been created """ + # Set sane defaults if not specified + kwargs.setdefault('display_name', "Server %s" % kwargs['internal_id']) instance_ref = self.db.instance_create(context, kwargs) inst_id = instance_ref['id'] From f53f5880c08994d04a552a41ce6f88dfbd867946 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 15:53:27 -0600 Subject: [PATCH 040/119] Oops, internal_id isn't available until after a save. This code saves twice; if I moved it into the DB layer we could do it in one save. However, we're moving to one sqlite db per compute worker, so I'd rather have two saves in order to keep the logic in the right layer. --- nova/compute/manager.py | 8 ++++++-- nova/db/sqlalchemy/api.py | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 0893db9fc5..6fc5c51866 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -99,10 +99,14 @@ class ComputeManager(manager.Manager): that has just been created """ - # Set sane defaults if not specified - kwargs.setdefault('display_name', "Server %s" % kwargs['internal_id']) instance_ref = self.db.instance_create(context, kwargs) inst_id = instance_ref['id'] + # Set sane defaults if not specified + if 'display_name' not in kwargs: + display_name = "Server %s" % instance_ref['internal_id'] + instance_ref['display_name'] = display_name + self.db.instance_update(context, inst_id, + { 'display_name': display_name }) elevated = context.elevated() if not security_groups: diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index afa55fc03a..dd96490545 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -530,6 +530,12 @@ def fixed_ip_update(context, address, values): #functions between the two of them as well. @require_context def instance_create(context, values): + """Create a new Instance record in the database. + + context - request context object + values - dict containing column values. + 'internal_id' is auto-generated and should not be specified. + """ instance_ref = models.Instance() instance_ref.update(values) From 8af2b1c97903f11034a95894a23bb7e77f573aa6 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 16:04:04 -0600 Subject: [PATCH 041/119] Going for a record commits per line changes ratio --- nova/compute/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6fc5c51866..e826bdaa2b 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -106,7 +106,7 @@ class ComputeManager(manager.Manager): display_name = "Server %s" % instance_ref['internal_id'] instance_ref['display_name'] = display_name self.db.instance_update(context, inst_id, - { 'display_name': display_name }) + {'display_name': display_name}) elevated = context.elevated() if not security_groups: From 3a96066987e42382e1ed75735473971ea90a76a1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 1 Dec 2010 22:43:46 +0000 Subject: [PATCH 042/119] fix nova.sh to reflect new location of ppa --- contrib/nova.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 1a9f93a3b1..f3eb8351a7 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -70,7 +70,7 @@ fi # You should only have to run this once if [ "$CMD" == "install" ]; then sudo apt-get install -y python-software-properties - sudo add-apt-repository ppa:nova-core/ppa + sudo add-apt-repository ppa:nova-core/trunk sudo apt-get update sudo apt-get install -y dnsmasq kpartx kvm gawk iptables ebtables sudo apt-get install -y user-mode-linux kvm libvirt-bin From fd44f9d2ec1d101960642a68d45bffc9c37f0d7f Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 2 Dec 2010 12:13:56 +0000 Subject: [PATCH 043/119] moved flags into xenapi/novadeps.py --- nova/virt/xenapi/novadeps.py | 40 ++++++++++++++++++++++++++++++++++++ nova/virt/xenapi_conn.py | 31 ++++++---------------------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/nova/virt/xenapi/novadeps.py b/nova/virt/xenapi/novadeps.py index ba62468fbc..985998486d 100644 --- a/nova/virt/xenapi/novadeps.py +++ b/nova/virt/xenapi/novadeps.py @@ -33,6 +33,46 @@ XENAPI_POWER_STATE = { 'Crashed': power_state.CRASHED} +flags.DEFINE_string('xenapi_connection_url', + None, + 'URL for connection to XenServer/Xen Cloud Platform.' + ' Required if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_username', + 'root', + 'Username for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_password', + None, + 'Password for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_float('xenapi_task_poll_interval', + 0.5, + 'The interval used for polling of remote tasks ' + '(Async.VM.start, etc). Used only if ' + 'connection_type=xenapi.') + + +class Configuration(object): + def __init__(self): + self._flags = flags.FLAGS + + @property + def xenapi_connection_url(self): + return self._flags.xenapi_connection_url + + @property + def xenapi_connection_username(self): + return self._flags.xenapi_connection_username + + @property + def xenapi_connection_password(self): + return self._flags.xenapi_connection_password + + @property + def xenapi_task_poll_interval(self): + return self._flags.xenapi_task_poll_interval + + class Instance(object): @classmethod diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 0a73b47741..51091ab19e 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -53,33 +53,14 @@ import xmlrpclib from twisted.internet import defer from twisted.internet import reactor -from nova import flags from nova import utils from xenapi.vmops import VMOps from xenapi.volumeops import VolumeOps +from xenapi.novadeps import Configuration XenAPI = None - - -FLAGS = flags.FLAGS -flags.DEFINE_string('xenapi_connection_url', - None, - 'URL for connection to XenServer/Xen Cloud Platform.' - ' Required if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_username', - 'root', - 'Username for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_password', - None, - 'Password for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_float('xenapi_task_poll_interval', - 0.5, - 'The interval used for polling of remote tasks ' - '(Async.VM.start, etc). Used only if ' - 'connection_type=xenapi.') +Config = Configuration() def get_connection(_): @@ -90,9 +71,9 @@ def get_connection(_): global XenAPI if XenAPI is None: XenAPI = __import__('XenAPI') - url = FLAGS.xenapi_connection_url - username = FLAGS.xenapi_connection_username - password = FLAGS.xenapi_connection_password + url = Config.xenapi_connection_url + username = Config.xenapi_connection_username + password = Config.xenapi_connection_password if not url or password is None: raise Exception('Must specify xenapi_connection_url, ' 'xenapi_connection_username (optionally), and ' @@ -177,7 +158,7 @@ class XenAPISession(object): #logging.debug('Polling task %s...', task) status = self._session.xenapi.task.get_status(task) if status == 'pending': - reactor.callLater(FLAGS.xenapi_task_poll_interval, + reactor.callLater(Config.xenapi_task_poll_interval(), self._poll_task, task, deferred) elif status == 'success': result = self._session.xenapi.task.get_result(task) From b684bc26fc7c7f41cf90e0294af35b2bda243733 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 2 Dec 2010 12:36:05 +0000 Subject: [PATCH 044/119] typo fix --- nova/virt/xenapi_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 51091ab19e..948fade7e9 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -158,7 +158,7 @@ class XenAPISession(object): #logging.debug('Polling task %s...', task) status = self._session.xenapi.task.get_status(task) if status == 'pending': - reactor.callLater(Config.xenapi_task_poll_interval(), + reactor.callLater(Config.xenapi_task_poll_interval, self._poll_task, task, deferred) elif status == 'success': result = self._session.xenapi.task.get_result(task) From 1e050bb4a8eeb65a7ac25a9fb90493567b5b07f4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 2 Dec 2010 15:18:45 +0100 Subject: [PATCH 045/119] Add a helpful error message to nova-manage in case of NoMoreNetworks. This is one of the most common problems people have, and the solution is not currently easily discoverable. This should address that. --- bin/nova-manage | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index eb7c6b87b5..62eec8353f 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -359,9 +359,14 @@ class ProjectCommands(object): def zipfile(self, project_id, user_id, filename='nova.zip'): """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - zip_file = self.manager.get_credentials(user_id, project_id) - with open(filename, 'w') as f: - f.write(zip_file) + try: + zip_file = self.manager.get_credentials(user_id, project_id) + with open(filename, 'w') as f: + f.write(zip_file) + except db.api.NoMoreNetworks: + print ('No more networks available. If this is a new ' + 'installation, you need\nto call something like this:\n\n' + ' nova-manage network create 10.0.0.0/8 10 64\n\n') class FloatingIpCommands(object): From 3af6da1fa5a38c8238ea45a7b03a6e3fbb78fe5b Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 10:08:56 -0600 Subject: [PATCH 046/119] Default Instance.display_name to a value even when None is explicitly passed in. --- nova/compute/manager.py | 2 +- nova/tests/compute_unittest.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index e826bdaa2b..c4a90e6043 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -102,7 +102,7 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_create(context, kwargs) inst_id = instance_ref['id'] # Set sane defaults if not specified - if 'display_name' not in kwargs: + if kwargs.get('display_name') is None: display_name = "Server %s" % instance_ref['internal_id'] instance_ref['display_name'] = display_name self.db.instance_update(context, inst_id, diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 71a1a4457d..85992b48cd 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -66,6 +66,16 @@ class ComputeTestCase(test.TrialTestCase): inst['ami_launch_index'] = 0 return db.instance_create(self.context, inst)['id'] + def test_create_instance_defaults_display_name(self): + """Verify that an instance cannot be created without a display_name.""" + cases = [dict(), dict(display_name=None)] + for instance in cases: + ref = self.compute.create_instance(self.context, None, **instance) + try: + self.assertNotEqual(ref.display_name, None) + finally: + db.instance_destroy(self.context, ref['id']) + def test_create_instance_associates_security_groups(self): """Make sure create_instance associates security groups""" inst = {} From 26571952bb8f1015b11d6b9514d232ad8a20d837 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 2 Dec 2010 10:21:43 -0800 Subject: [PATCH 047/119] Moved reboot/rescue methods into nova.compute.api. --- nova/api/cloud.py | 58 ----------------------------------- nova/api/ec2/cloud.py | 7 ++--- nova/api/openstack/servers.py | 3 +- nova/compute/api.py | 27 ++++++++++++++++ 4 files changed, 31 insertions(+), 64 deletions(-) delete mode 100644 nova/api/cloud.py diff --git a/nova/api/cloud.py b/nova/api/cloud.py deleted file mode 100644 index b8f15019f4..0000000000 --- a/nova/api/cloud.py +++ /dev/null @@ -1,58 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Methods for API calls to control instances via AMQP. -""" - - -from nova import db -from nova import flags -from nova import rpc - -FLAGS = flags.FLAGS - - -def reboot(instance_id, context=None): - """Reboot the given instance.""" - instance_ref = db.instance_get_by_internal_id(context, instance_id) - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "reboot_instance", - "args": {"instance_id": instance_ref['id']}}) - - -def rescue(instance_id, context): - """Rescue the given instance.""" - instance_ref = db.instance_get_by_internal_id(context, instance_id) - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "rescue_instance", - "args": {"instance_id": instance_ref['id']}}) - - -def unrescue(instance_id, context): - """Unrescue the given instance.""" - instance_ref = db.instance_get_by_internal_id(context, instance_id) - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "unrescue_instance", - "args": {"instance_id": instance_ref['id']}}) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index e50906ae17..161d2d038d 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -41,7 +41,6 @@ from nova import rpc from nova import utils from nova.compute import api as compute_api from nova.compute import instance_types -from nova.api import cloud from nova.image.s3 import S3ImageService @@ -834,19 +833,19 @@ class CloudController(object): """instance_id is a list of instance ids""" for ec2_id in instance_id: internal_id = ec2_id_to_internal_id(ec2_id) - cloud.reboot(internal_id, context=context) + self.compute_api.reboot(context, internal_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" internal_id = ec2_id_to_internal_id(instance_id) - cloud.rescue(internal_id, context=context) + self.compute_api.rescue(context, internal_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" internal_id = ec2_id_to_internal_id(instance_id) - cloud.unrescue(internal_id, context=context) + self.compute_api.unrescue(context, internal_id) return True def update_instance(self, context, ec2_id, **kwargs): diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 11170bbf5e..d34dd78fb0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -25,7 +25,6 @@ from nova import rpc from nova import utils from nova import wsgi from nova import context -from nova.api import cloud from nova.api.openstack import faults from nova.compute import api as compute_api from nova.compute import instance_types @@ -191,7 +190,7 @@ class Controller(wsgi.Controller): inst_ref = self.db.instance_get_by_internal_id(ctxt, int(id)) if not inst_ref or (inst_ref and not inst_ref.user_id == user_id): return faults.Fault(exc.HTTPUnprocessableEntity()) - cloud.reboot(id) + self.compute_api.reboot(ctxt, id) def _get_network_topic(self, context): """Retrieves the network host for a project""" diff --git a/nova/compute/api.py b/nova/compute/api.py index 929342a1e9..da01ca61ad 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -210,3 +210,30 @@ class ComputeAPI(base.Base): """ self.db.instance_update(context, instance_id, kwargs) + + def reboot(self, context, instance_id): + """Reboot the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "reboot_instance", + "args": {"instance_id": instance['id']}}) + + def rescue(self, context, instance_id): + """Rescue the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "rescue_instance", + "args": {"instance_id": instance['id']}}) + + def unrescue(self, context, instance_id): + """Unrescue the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "unrescue_instance", + "args": {"instance_id": instance['id']}}) From 111285b9bcbee26e3f49b92dcc68355e251007a7 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 2 Dec 2010 10:53:32 -0800 Subject: [PATCH 048/119] Added test files to be ignored. --- .bzrignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.bzrignore b/.bzrignore index ab099d3e32..82db46fa22 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1,3 +1,13 @@ run_tests.err.log .nova-venv ChangeLog +_trial_temp +keys +networks +nova.sqlite +CA/cacert.pem +CA/index.txt* +CA/openssl.cnf +CA/serial* +CA/newcerts/*.pem +CA/private/cakey.pem From 9d5e1b52f837047aac55d08a664a35be7cc5b8ef Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 12:58:13 -0600 Subject: [PATCH 049/119] Correctly translate instance ids to internal_ids in some spots we neglected. And do some pylint cleanup. --- nova/api/openstack/servers.py | 12 ++++++------ nova/compute/manager.py | 10 ++++------ nova/virt/xenapi.py | 2 -- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 11170bbf5e..f85aabbfaa 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -15,8 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import time - import webob from webob import exc @@ -64,7 +62,7 @@ def _entity_detail(inst): inst_dict = {} mapped_keys = dict(status='state', imageId='image_id', - flavorId='instance_type', name='display_name', id='id') + flavorId='instance_type', name='display_name', id='internal_id') for k, v in mapped_keys.iteritems(): inst_dict[k] = inst[v] @@ -79,7 +77,7 @@ def _entity_detail(inst): def _entity_inst(inst): """ Filters all model attributes save for id and name """ - return dict(server=dict(id=inst['id'], name=inst['display_name'])) + return dict(server=dict(id=inst['internal_id'], name=inst['display_name'])) class Controller(wsgi.Controller): @@ -89,7 +87,7 @@ class Controller(wsgi.Controller): 'application/xml': { "attributes": { "server": ["id", "imageId", "name", "flavorId", "hostId", - "status", "progress", "progress"]}}} + "status", "progress"]}}} def __init__(self, db_driver=None): if not db_driver: @@ -176,7 +174,7 @@ class Controller(wsgi.Controller): self.db_driver.instance_update(ctxt, int(id), _filter_params(inst_dict['server'])) - return faults.Fault(exc.HTTPNoContent()) + return exc.HTTPNoContent() def action(self, req, id): """ multi-purpose method used to reboot, rebuild, and @@ -191,6 +189,8 @@ class Controller(wsgi.Controller): inst_ref = self.db.instance_get_by_internal_id(ctxt, int(id)) if not inst_ref or (inst_ref and not inst_ref.user_id == user_id): return faults.Fault(exc.HTTPUnprocessableEntity()) + #TODO(gundlach): pass reboot_type, support soft reboot in + #virt driver cloud.reboot(id) def _get_network_topic(self, context): diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b5eb23b240..dd8d41129c 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -22,8 +22,8 @@ Handles all processes relating to instances (guest vms). The :py:class:`ComputeManager` class is a :py:class:`nova.manager.Manager` that handles RPC calls relating to creating instances. It is responsible for building a disk image, launching it via the underlying virtualization driver, -responding to calls to check it state, attaching persistent as well as -termination. +responding to calls to check its state, attaching persistent storage, and +terminating it. **Related Flags** @@ -39,7 +39,6 @@ import logging from twisted.internet import defer -from nova import db from nova import exception from nova import flags from nova import manager @@ -50,10 +49,11 @@ FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', 'where instances are stored on disk') flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', - 'Driver to use for volume creation') + 'Driver to use for controlling virtualization') class ComputeManager(manager.Manager): + """Manages the running instances from creation to destruction.""" def __init__(self, compute_driver=None, *args, **kwargs): @@ -93,7 +93,6 @@ class ComputeManager(manager.Manager): if instance_ref['name'] in self.driver.list_instances(): raise exception.Error("Instance has already been created") logging.debug("instance %s: starting...", instance_id) - project_id = instance_ref['project_id'] self.network_manager.setup_compute_network(context, instance_id) self.db.instance_update(context, instance_id, @@ -135,7 +134,6 @@ class ComputeManager(manager.Manager): self.db.instance_destroy(context, instance_id) raise exception.Error('trying to destroy already destroyed' ' instance: %s' % instance_id) - yield self.driver.destroy(instance_ref) # TODO(ja): should we keep it in a terminated state for a bit? diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 3169562a53..de3d685829 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -52,11 +52,9 @@ import xmlrpclib from twisted.internet import defer from twisted.internet import reactor -from twisted.internet import task from nova import db from nova import flags -from nova import process from nova import utils from nova.auth.manager import AuthManager from nova.compute import instance_types From 7bcbc2a6e1b907886e03e5254dcd0a726ccdcd9d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 13:29:37 -0600 Subject: [PATCH 050/119] Oops, update 'display_name', not 'name'. And un-extract-method. --- nova/api/openstack/__init__.py | 2 ++ nova/api/openstack/servers.py | 20 +++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 1dd3ba770e..4ca108c4e7 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -25,6 +25,7 @@ import time import logging import routes +import traceback import webob.dec import webob.exc import webob @@ -61,6 +62,7 @@ class API(wsgi.Middleware): return req.get_response(self.application) except Exception as ex: logging.warn("Caught error: %s" % str(ex)) + logging.debug(traceback.format_exc()) exc = webob.exc.HTTPInternalServerError(explanation=str(ex)) return faults.Fault(exc) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f85aabbfaa..a2a637def8 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -34,16 +34,6 @@ import nova.image.service FLAGS = flags.FLAGS -def _filter_params(inst_dict): - """ Extracts all updatable parameters for a server update request """ - keys = dict(name='name', admin_pass='adminPass') - new_attrs = {} - for k, v in keys.items(): - if v in inst_dict: - new_attrs[k] = inst_dict[v] - return new_attrs - - def _entity_list(entities): """ Coerces a list of servers into proper dictionary format """ return dict(servers=entities) @@ -171,9 +161,13 @@ class Controller(wsgi.Controller): if not instance or instance.user_id != user_id: return faults.Fault(exc.HTTPNotFound()) - self.db_driver.instance_update(ctxt, - int(id), - _filter_params(inst_dict['server'])) + update_dict = {} + if 'adminPass' in inst_dict['server']: + update_dict['admin_pass'] = inst_dict['server']['adminPass'] + if 'name' in inst_dict['server']: + update_dict['display_name'] = inst_dict['server']['name'] + + self.db_driver.instance_update(ctxt, instance['id'], update_dict) return exc.HTTPNoContent() def action(self, req, id): From a8df0a7d6c2de55d7906fa311f79887ccf575508 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Thu, 2 Dec 2010 13:48:39 -0600 Subject: [PATCH 051/119] Fixing single node install doc --- doc/source/adminguide/multi.node.install.rst | 13 +++++------- doc/source/adminguide/single.node.install.rst | 20 +++++++++++++++---- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst index dcceb539bf..1eed30c5b3 100644 --- a/doc/source/adminguide/multi.node.install.rst +++ b/doc/source/adminguide/multi.node.install.rst @@ -19,7 +19,7 @@ Installing Nova on Multiple Servers =================================== When you move beyond evaluating the technology and into building an actual -production environemnt, you will need to know how to configure your datacenter +production environment, you will need to know how to configure your datacenter and how to deploy components across your clusters. This guide should help you through that process. @@ -161,7 +161,7 @@ Step 3 Setup the sql db GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; SET PASSWORD FOR 'root'@'%' = PASSWORD('nova'); -7. branch and install Nova +7. Branch and install Nova :: @@ -186,9 +186,7 @@ Step 4 Setup Nova environment Note: The nova-manage service assumes that the first IP address is your network (like 192.168.0.0), that the 2nd IP is your gateway (192.168.0.1), and that the broadcast is the very last IP in the range you defined (192.168.0.255). If this is not the case you will need to manually edit the sql db 'networks' table.o. -On running this command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. We ended up doing this manually, (update query fired directly in the DB). Is there a better way to mark a network as bridged? - -Update: This has been resolved w.e.f 27/10. network is marked as bridged automatically based on the type of n/w manager selected. +On running this command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. The Network is marked as bridged automatically based on the type of network manager selected. More networking details to create a network bridge for flat network ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -233,7 +231,6 @@ unzip them in your home directory, and add them to your environment:: echo ". creds/novarc" >> ~/.bashrc ~/.bashrc - Step 6 Restart all relevant services ------------------------------------ @@ -249,8 +246,8 @@ Restart relevant nova services:: .. todo:: do we still need the content below? -Bare-metal Provisioning ------------------------ +Bare-metal Provisioning Notes +----------------------------- To install the base operating system you can use PXE booting. diff --git a/doc/source/adminguide/single.node.install.rst b/doc/source/adminguide/single.node.install.rst index 27597962aa..f6b2290bcf 100644 --- a/doc/source/adminguide/single.node.install.rst +++ b/doc/source/adminguide/single.node.install.rst @@ -63,8 +63,20 @@ You see an access key and a secret key export, such as these made-up ones::: export EC2_ACCESS_KEY=4e6498a2-blah-blah-blah-17d1333t97fd export EC2_SECRET_KEY=0a520304-blah-blah-blah-340sp34k05bbe9a7 +Step 5: Create the network +-------------------------- -Step 5: Create a project with the user you created +Type or copy/paste in the following line to create a network prior to creating a project. + +:: + + sudo nova-manage network create 10.0.0.0/8 1 64 + +For this command, the IP address is the cidr notation for your netmask, such as 192.168.1.0/24. The value 1 is the total number of networks you want made, and the 64 value is the total number of ips in all networks. + +After running this command, entries are made in the 'networks' and 'fixed_ips' table in the database. + +Step 6: Create a project with the user you created -------------------------------------------------- Type or copy/paste in the following line to create a project named IRT (for Ice Road Truckers, of course) with the newly-created user named anne. @@ -94,7 +106,7 @@ Type or copy/paste in the following line to create a project named IRT (for Ice Data Base Updated -Step 6: Unzip the nova.zip +Step 7: Unzip the nova.zip -------------------------- You should have a nova.zip file in your current working directory. Unzip it with this command: @@ -116,7 +128,7 @@ You'll see these files extract. extracting: cacert.pem -Step 7: Source the rc file +Step 8: Source the rc file -------------------------- Type or copy/paste the following to source the novarc file in your current working directory. @@ -125,7 +137,7 @@ Type or copy/paste the following to source the novarc file in your current worki . novarc -Step 8: Pat yourself on the back :) +Step 9: Pat yourself on the back :) ----------------------------------- Congratulations, your cloud is up and running, you’ve created an admin user, retrieved the user's credentials and put them in your environment. From 84b130f5fcc02964bc38423bb0153db9cc89e520 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 14:14:31 -0600 Subject: [PATCH 052/119] Update tests to use proper id --- nova/tests/api/openstack/test_servers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 2eee4e5062..8060995ad0 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -48,8 +48,8 @@ def return_security_group(context, instance_id, security_group_id): def stub_instance(id, user_id=1): - return Instance(id=id, state=0, image_id=10, display_name='server%s' % id, - user_id=user_id) + return Instance(id=id+123456, state=0, image_id=10, user_id=user_id, + display_name='server%s' % id, internal_id=id) class ServersTest(unittest.TestCase): From 8be00510243918a67558b60557e7261e4649e94e Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 14:17:41 -0600 Subject: [PATCH 053/119] Use newfangled compute_api --- nova/api/openstack/servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a2a637def8..e7f765c028 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -167,7 +167,7 @@ class Controller(wsgi.Controller): if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] - self.db_driver.instance_update(ctxt, instance['id'], update_dict) + self.compute_api.update_instance(ctxt, instance['id'], update_dict) return exc.HTTPNoContent() def action(self, req, id): From ad8577fdf07cc6ef8734962c93c85cb03afe23a7 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 15:33:43 -0600 Subject: [PATCH 054/119] pep8 --- nova/tests/api/openstack/test_servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 8060995ad0..44ac8f3427 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -48,7 +48,7 @@ def return_security_group(context, instance_id, security_group_id): def stub_instance(id, user_id=1): - return Instance(id=id+123456, state=0, image_id=10, user_id=user_id, + return Instance(id=id + 123456, state=0, image_id=10, user_id=user_id, display_name='server%s' % id, internal_id=id) From 47b47bc4ae34f90a6d1c59718b5ee759fb7c7327 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 2 Dec 2010 15:26:14 -0800 Subject: [PATCH 055/119] Pushed terminate instance and network manager/topic methods into network.compute.api. --- nova/api/ec2/cloud.py | 65 ++----------------- nova/api/openstack/servers.py | 26 ++------ nova/compute/api.py | 82 +++++++++++++++++++++--- nova/tests/api/openstack/test_servers.py | 13 ++++ 4 files changed, 98 insertions(+), 88 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 161d2d038d..7978e08a04 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -94,7 +94,7 @@ class CloudController(object): """ def __init__(self): self.network_manager = utils.import_object(FLAGS.network_manager) - self.compute_api = compute_api.ComputeAPI() + self.compute_api = compute_api.ComputeAPI(self.network_manager) self.image_service = S3ImageService() self.setup() @@ -752,7 +752,6 @@ class CloudController(object): instance_types.get_by_type(kwargs.get('instance_type', None)), self.image_service, kwargs['image_id'], - self._get_network_topic(context), min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), @@ -768,65 +767,11 @@ class CloudController(object): def terminate_instances(self, context, instance_id, **kwargs): """Terminate each instance in instance_id, which is a list of ec2 ids. - - instance_id is a kwarg so its name cannot be modified. - """ - ec2_id_list = instance_id + instance_id is a kwarg so its name cannot be modified.""" logging.debug("Going to start terminating instances") - for id_str in ec2_id_list: - internal_id = ec2_id_to_internal_id(id_str) - logging.debug("Going to try and terminate %s" % id_str) - try: - instance_ref = db.instance_get_by_internal_id(context, - internal_id) - except exception.NotFound: - logging.warning("Instance %s was not found during terminate", - id_str) - continue - - if (instance_ref['state_description'] == 'terminating'): - logging.warning("Instance %s is already being terminated", - id_str) - continue - now = datetime.datetime.utcnow() - self.compute_api.update_instance(context, - instance_ref['id'], - state_description='terminating', - state=0, - terminated_at=now) - - # FIXME(ja): where should network deallocate occur? - address = db.instance_get_floating_address(context, - instance_ref['id']) - if address: - logging.debug("Disassociating address %s" % address) - # NOTE(vish): Right now we don't really care if the ip is - # disassociated. We may need to worry about - # checking this later. Perhaps in the scheduler? - network_topic = self._get_network_topic(context) - rpc.cast(context, - network_topic, - {"method": "disassociate_floating_ip", - "args": {"floating_address": address}}) - - address = db.instance_get_fixed_address(context, - instance_ref['id']) - if address: - logging.debug("Deallocating address %s" % address) - # NOTE(vish): Currently, nothing needs to be done on the - # network node until release. If this changes, - # we will need to cast here. - self.network_manager.deallocate_fixed_ip(context.elevated(), - address) - - host = instance_ref['host'] - if host: - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "terminate_instance", - "args": {"instance_id": instance_ref['id']}}) - else: - db.instance_destroy(context, instance_ref['id']) + for ec2_id in instance_id: + internal_id = ec2_id_to_internal_id(ec2_id) + self.compute_api.delete_instance(context, internal_id) return True def reboot_instances(self, context, instance_id, **kwargs): diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index d34dd78fb0..1d93f783cf 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -20,11 +20,12 @@ import time import webob from webob import exc +from nova import context +from nova import exception from nova import flags from nova import rpc from nova import utils from nova import wsgi -from nova import context from nova.api.openstack import faults from nova.compute import api as compute_api from nova.compute import instance_types @@ -94,7 +95,6 @@ class Controller(wsgi.Controller): if not db_driver: db_driver = FLAGS.db_driver self.db_driver = utils.import_object(db_driver) - self.network_manager = utils.import_object(FLAGS.network_manager) self.compute_api = compute_api.ComputeAPI() super(Controller, self).__init__() @@ -132,11 +132,11 @@ class Controller(wsgi.Controller): """ Destroys a server """ user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) - instance = self.db_driver.instance_get_by_internal_id(ctxt, int(id)) - if instance and instance['user_id'] == user_id: - self.db_driver.instance_destroy(ctxt, id) - return faults.Fault(exc.HTTPAccepted()) - return faults.Fault(exc.HTTPNotFound()) + try: + self.compute_api.delete_instance(ctxt, int(id)) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) + return faults.Fault(exc.HTTPAccepted()) def create(self, req): """ Creates a new server for a given user """ @@ -151,7 +151,6 @@ class Controller(wsgi.Controller): instance_types.get_by_flavor_id(env['server']['flavorId']), utils.import_object(FLAGS.image_service), env['server']['imageId'], - self._get_network_topic(ctxt), name=env['server']['name'], description=env['server']['name'], key_name=key_pair['name'], @@ -191,14 +190,3 @@ class Controller(wsgi.Controller): if not inst_ref or (inst_ref and not inst_ref.user_id == user_id): return faults.Fault(exc.HTTPUnprocessableEntity()) self.compute_api.reboot(ctxt, id) - - def _get_network_topic(self, context): - """Retrieves the network host for a project""" - network_ref = self.network_manager.get_network(context) - host = network_ref['host'] - if not host: - host = rpc.call(context, - FLAGS.network_topic, - {"method": "set_network_host", - "args": {"network_id": network_ref['id']}}) - return self.db_driver.queue_get_for(context, FLAGS.network_topic, host) diff --git a/nova/compute/api.py b/nova/compute/api.py index da01ca61ad..457d6e27fc 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -20,6 +20,7 @@ Handles all API requests relating to instances (guest vms). """ +import datetime import logging import time @@ -43,17 +44,17 @@ def generate_default_hostname(internal_id): class ComputeAPI(base.Base): """API for interacting with the compute manager.""" - def __init__(self, **kwargs): - self.network_manager = utils.import_object(FLAGS.network_manager) + def __init__(self, network_manager=None, **kwargs): + if not network_manager: + network_manager = utils.import_object(FLAGS.network_manager) + self.network_manager = network_manager super(ComputeAPI, self).__init__(**kwargs) - # TODO(eday): network_topic arg should go away once we push network - # allocation into the scheduler or compute worker. def create_instances(self, context, instance_type, image_service, image_id, - network_topic, min_count=1, max_count=1, - kernel_id=None, ramdisk_id=None, name='', - description='', user_data='', key_name=None, - key_data=None, security_group='default', + min_count=1, max_count=1, kernel_id=None, + ramdisk_id=None, name='', description='', + user_data='', key_name=None, key_data=None, + security_group='default', generate_hostname=generate_default_hostname): """Create the number of instances requested if quote and other arguments check out ok.""" @@ -139,7 +140,7 @@ class ComputeAPI(base.Base): instance_id, is_vpn) rpc.cast(elevated, - network_topic, + self._get_network_topic(context), {"method": "setup_fixed_ip", "args": {"address": address}}) @@ -211,6 +212,58 @@ class ComputeAPI(base.Base): """ self.db.instance_update(context, instance_id, kwargs) + def delete_instance(self, context, instance_id): + logging.debug("Going to try and terminate %d" % instance_id) + try: + instance = self.db.instance_get_by_internal_id(context, + instance_id) + except exception.NotFound as e: + logging.warning("Instance %d was not found during terminate", + instance_id) + raise e + + if (instance['state_description'] == 'terminating'): + logging.warning("Instance %d is already being terminated", + instance_id) + return + + self.update_instance(context, + instance['id'], + state_description='terminating', + state=0, + terminated_at=datetime.datetime.utcnow()) + + # FIXME(ja): where should network deallocate occur? + address = self.db.instance_get_floating_address(context, + instance['id']) + if address: + logging.debug("Disassociating address %s" % address) + # NOTE(vish): Right now we don't really care if the ip is + # disassociated. We may need to worry about + # checking this later. Perhaps in the scheduler? + rpc.cast(context, + self._get_network_topic(context), + {"method": "disassociate_floating_ip", + "args": {"floating_address": address}}) + + address = self.db.instance_get_fixed_address(context, instance['id']) + if address: + logging.debug("Deallocating address %s" % address) + # NOTE(vish): Currently, nothing needs to be done on the + # network node until release. If this changes, + # we will need to cast here. + self.network_manager.deallocate_fixed_ip(context.elevated(), + address) + + host = instance['host'] + if host: + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "terminate_instance", + "args": {"instance_id": instance['id']}}) + else: + self.db.instance_destroy(context, instance['id']) + def reboot(self, context, instance_id): """Reboot the given instance.""" instance = self.db.instance_get_by_internal_id(context, instance_id) @@ -237,3 +290,14 @@ class ComputeAPI(base.Base): self.db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "unrescue_instance", "args": {"instance_id": instance['id']}}) + + def _get_network_topic(self, context): + """Retrieves the network host for a project""" + network_ref = self.network_manager.get_network(context) + host = network_ref['host'] + if not host: + host = rpc.call(context, + FLAGS.network_topic, + {"method": "set_network_host", + "args": {"network_id": network_ref['id']}}) + return self.db.queue_get_for(context, FLAGS.network_topic, host) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 2eee4e5062..aebb3d1b54 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -47,6 +47,14 @@ def return_security_group(context, instance_id, security_group_id): pass +def instance_update(context, instance_id, kwargs): + pass + + +def instance_address(context, instance_id): + return None + + def stub_instance(id, user_id=1): return Instance(id=id, state=0, image_id=10, display_name='server%s' % id, user_id=user_id) @@ -69,6 +77,11 @@ class ServersTest(unittest.TestCase): return_servers) self.stubs.Set(nova.db.api, 'instance_add_security_group', return_security_group) + self.stubs.Set(nova.db.api, 'instance_update', instance_update) + self.stubs.Set(nova.db.api, 'instance_get_fixed_address', + instance_address) + self.stubs.Set(nova.db.api, 'instance_get_floating_address', + instance_address) def tearDown(self): self.stubs.UnsetAll() From 108bab90cb70798151b8e6a09d2176a3eb120380 Mon Sep 17 00:00:00 2001 From: Ryan Lucio Date: Thu, 2 Dec 2010 17:01:44 -0800 Subject: [PATCH 056/119] Updated sqlalchemy model to make the internal_id column of the instances table as unsigned integer --- nova/db/sqlalchemy/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index fe0a9a9216..18ba80caf6 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -27,6 +27,7 @@ from sqlalchemy import ForeignKey, DateTime, Boolean, Text from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.schema import ForeignKeyConstraint +from sqlalchemy.databases import mysql from nova.db.sqlalchemy.session import get_session @@ -155,7 +156,7 @@ class Instance(BASE, NovaBase): """Represents a guest vm.""" __tablename__ = 'instances' id = Column(Integer, primary_key=True) - internal_id = Column(Integer, unique=True) + internal_id = Column(mysql.MSInteger(unsigned=True), unique=True) admin_pass = Column(String(255)) From 4203aa1060e5a97bed86d2e201c4c2443ef7e042 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Fri, 3 Dec 2010 12:21:18 -0800 Subject: [PATCH 057/119] Finished cleaning up the openstack servers API, it no longer touches the database directly. Also cleaned up similar things in ec2 API and refactored a couple methods in nova.compute.api to accomodate this work. --- nova/api/ec2/cloud.py | 25 +++--- nova/api/openstack/servers.py | 48 +++++------- nova/auth/manager.py | 4 + nova/compute/api.py | 96 +++++++++++------------- nova/db/sqlalchemy/api.py | 1 + nova/flags.py | 2 +- nova/tests/api/openstack/fakes.py | 3 +- nova/tests/api/openstack/test_servers.py | 10 +-- nova/tests/compute_unittest.py | 24 +++--- 9 files changed, 94 insertions(+), 119 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 7978e08a04..4eef5e1efb 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -41,7 +41,6 @@ from nova import rpc from nova import utils from nova.compute import api as compute_api from nova.compute import instance_types -from nova.image.s3 import S3ImageService FLAGS = flags.FLAGS @@ -94,8 +93,9 @@ class CloudController(object): """ def __init__(self): self.network_manager = utils.import_object(FLAGS.network_manager) - self.compute_api = compute_api.ComputeAPI(self.network_manager) - self.image_service = S3ImageService() + self.image_service = utils.import_object(FLAGS.image_service) + self.compute_api = compute_api.ComputeAPI(self.network_manager, + self.image_service) self.setup() def __str__(self): @@ -119,7 +119,7 @@ class CloudController(object): def _get_mpi_data(self, context, project_id): result = {} - for instance in db.instance_get_all_by_project(context, project_id): + for instance in self.compute_api.get_instances(context, project_id): if instance['fixed_ip']: line = '%s slots=%d' % (instance['fixed_ip']['address'], instance['vcpus']) @@ -438,7 +438,7 @@ class CloudController(object): # instance_id is passed in as a list of instances ec2_id = instance_id[0] internal_id = ec2_id_to_internal_id(ec2_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) + instance_ref = self.compute_api.get_instance(context, internal_id) output = rpc.call(context, '%s.%s' % (FLAGS.compute_topic, instance_ref['host']), @@ -535,7 +535,7 @@ class CloudController(object): if volume_ref['attach_status'] == "attached": raise exception.ApiError("Volume is already attached") internal_id = ec2_id_to_internal_id(instance_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) + instance_ref = self.compute_api.get_instance(context, internal_id) host = instance_ref['host'] rpc.cast(context, db.queue_get_for(context, FLAGS.compute_topic, host), @@ -613,11 +613,7 @@ class CloudController(object): instances = db.instance_get_all_by_reservation(context, reservation_id) else: - if context.user.is_admin(): - instances = db.instance_get_all(context) - else: - instances = db.instance_get_all_by_project(context, - context.project_id) + instances = self.compute_api.get_instances(context) for instance in instances: if not context.user.is_admin(): if instance['image_id'] == FLAGS.vpn_image_id: @@ -714,7 +710,7 @@ class CloudController(object): def associate_address(self, context, instance_id, public_ip, **kwargs): internal_id = ec2_id_to_internal_id(instance_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) + instance_ref = self.compute_api.get_instance(context, internal_id) fixed_address = db.instance_get_fixed_address(context, instance_ref['id']) floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) @@ -750,13 +746,12 @@ class CloudController(object): max_count = int(kwargs.get('max_count', 1)) instances = self.compute_api.create_instances(context, instance_types.get_by_type(kwargs.get('instance_type', None)), - self.image_service, kwargs['image_id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), ramdisk_id=kwargs.get('ramdisk_id'), - name=kwargs.get('display_name'), + display_name=kwargs.get('display_name'), description=kwargs.get('display_description'), user_data=kwargs.get('user_data', ''), key_name=kwargs.get('key_name'), @@ -801,7 +796,7 @@ class CloudController(object): changes[field] = kwargs[field] if changes: internal_id = ec2_id_to_internal_id(ec2_id) - inst = db.instance_get_by_internal_id(context, internal_id) + inst = self.compute_api.get_instance(context, internal_id) db.instance_update(context, inst['id'], kwargs) return True diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a9da148677..b644876b0c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -15,23 +15,17 @@ # License for the specific language governing permissions and limitations # under the License. -import webob from webob import exc from nova import context from nova import exception -from nova import flags -from nova import rpc -from nova import utils from nova import wsgi from nova.api.openstack import faults +from nova.auth import manager as auth_manager from nova.compute import api as compute_api from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack -import nova.image.service - -FLAGS = flags.FLAGS def _entity_list(entities): @@ -79,10 +73,7 @@ class Controller(wsgi.Controller): "server": ["id", "imageId", "name", "flavorId", "hostId", "status", "progress"]}}} - def __init__(self, db_driver=None): - if not db_driver: - db_driver = FLAGS.db_driver - self.db_driver = utils.import_object(db_driver) + def __init__(self): self.compute_api = compute_api.ComputeAPI() super(Controller, self).__init__() @@ -101,7 +92,7 @@ class Controller(wsgi.Controller): """ user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) - instance_list = self.db_driver.instance_get_all_by_user(ctxt, user_id) + instance_list = self.compute_api.get_instances(ctxt) limited_list = nova.api.openstack.limited(instance_list, req) res = [entity_maker(inst)['server'] for inst in limited_list] return _entity_list(res) @@ -110,7 +101,7 @@ class Controller(wsgi.Controller): """ Returns server details by server id """ user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) - inst = self.db_driver.instance_get_by_internal_id(ctxt, int(id)) + inst = self.compute_api.get_instance(ctxt, int(id)) if inst: if inst.user_id == user_id: return _entity_detail(inst) @@ -134,12 +125,11 @@ class Controller(wsgi.Controller): user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) - key_pair = self.db_driver.key_pair_get_all_by_user(None, user_id)[0] + key_pair = auth_manager.AuthManager.get_key_pairs(ctxt)[0] instances = self.compute_api.create_instances(ctxt, instance_types.get_by_flavor_id(env['server']['flavorId']), - utils.import_object(FLAGS.image_service), env['server']['imageId'], - name=env['server']['name'], + display_name=env['server']['name'], description=env['server']['name'], key_name=key_pair['name'], key_data=key_pair['public_key']) @@ -149,27 +139,24 @@ class Controller(wsgi.Controller): """ Updates the server name or password """ user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) - inst_dict = self._deserialize(req.body, req) - if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) - instance = self.db_driver.instance_get_by_internal_id(ctxt, int(id)) - if not instance or instance.user_id != user_id: - return faults.Fault(exc.HTTPNotFound()) - update_dict = {} if 'adminPass' in inst_dict['server']: update_dict['admin_pass'] = inst_dict['server']['adminPass'] if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] - self.compute_api.update_instance(ctxt, instance['id'], update_dict) + try: + self.compute_api.update_instance(ctxt, instance['id'], update_dict) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() def action(self, req, id): - """ multi-purpose method used to reboot, rebuild, and + """ Multi-purpose method used to reboot, rebuild, and resize a server """ user_id = req.environ['nova.context']['user']['id'] ctxt = context.RequestContext(user_id, user_id) @@ -177,10 +164,11 @@ class Controller(wsgi.Controller): try: reboot_type = input_dict['reboot']['type'] except Exception: - raise faults.Fault(webob.exc.HTTPNotImplemented()) - inst_ref = self.db.instance_get_by_internal_id(ctxt, int(id)) - if not inst_ref or (inst_ref and not inst_ref.user_id == user_id): + raise faults.Fault(exc.HTTPNotImplemented()) + try: + # TODO(gundlach): pass reboot_type, support soft reboot in + # virt driver + self.compute_api.reboot(ctxt, id) + except: return faults.Fault(exc.HTTPUnprocessableEntity()) - # TODO(gundlach): pass reboot_type, support soft reboot in - # virt driver - self.compute_api.reboot(ctxt, id) + return exc.HTTPNoContent() diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 7b2b681616..11c3bd6dfb 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -624,6 +624,10 @@ class AuthManager(object): with self.driver() as drv: drv.modify_user(uid, access_key, secret_key, admin) + @staticmethod + def get_key_pairs(context): + return db.key_pair_get_all_by_user(context.elevated(), context.user_id) + def get_credentials(self, user, project=None): """Get credential zip for user in project""" if not isinstance(user, User): diff --git a/nova/compute/api.py b/nova/compute/api.py index 457d6e27fc..995bed91bc 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -44,16 +44,19 @@ def generate_default_hostname(internal_id): class ComputeAPI(base.Base): """API for interacting with the compute manager.""" - def __init__(self, network_manager=None, **kwargs): + def __init__(self, network_manager=None, image_service=None, **kwargs): if not network_manager: network_manager = utils.import_object(FLAGS.network_manager) self.network_manager = network_manager + if not image_service: + image_service = utils.import_object(FLAGS.image_service) + self.image_service = image_service super(ComputeAPI, self).__init__(**kwargs) - def create_instances(self, context, instance_type, image_service, image_id, - min_count=1, max_count=1, kernel_id=None, - ramdisk_id=None, name='', description='', - user_data='', key_name=None, key_data=None, + def create_instances(self, context, instance_type, image_id, min_count=1, + max_count=1, kernel_id=None, ramdisk_id=None, + display_name='', description='', user_data='', + key_name=None, key_data=None, security_group='default', generate_hostname=generate_default_hostname): """Create the number of instances requested if quote and @@ -70,15 +73,15 @@ class ComputeAPI(base.Base): is_vpn = image_id == FLAGS.vpn_image_id if not is_vpn: - image = image_service.show(context, image_id) + image = self.image_service.show(context, image_id) if kernel_id is None: kernel_id = image.get('kernelId', FLAGS.default_kernel) if ramdisk_id is None: ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) # Make sure we have access to kernel and ramdisk - image_service.show(context, kernel_id) - image_service.show(context, ramdisk_id) + self.image_service.show(context, kernel_id) + self.image_service.show(context, ramdisk_id) if security_group is None: security_group = ['default'] @@ -111,7 +114,7 @@ class ComputeAPI(base.Base): 'memory_mb': type_data['memory_mb'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], - 'display_name': name, + 'display_name': display_name, 'display_description': description, 'key_name': key_name, 'key_data': key_data} @@ -123,14 +126,25 @@ class ComputeAPI(base.Base): instance = dict(mac_address=utils.generate_mac(), launch_index=num, **base_options) - instance_ref = self.create_instance(context, security_groups, - **instance) - instance_id = instance_ref['id'] - internal_id = instance_ref['internal_id'] - hostname = generate_hostname(internal_id) - self.update_instance(context, instance_id, hostname=hostname) - instances.append(dict(id=instance_id, internal_id=internal_id, - hostname=hostname, **instance)) + instance = self.db.instance_create(context, instance) + instance_id = instance['id'] + internal_id = instance['internal_id'] + + elevated = context.elevated() + if not security_groups: + security_groups = [] + for security_group_id in security_groups: + self.db.instance_add_security_group(elevated, + instance_id, + security_group_id) + + # Set sane defaults if not specified + updates = dict(hostname=generate_hostname(internal_id)) + if 'display_name' not in instance: + updates['display_name'] = "Server %s" % internal_id + + instance = self.update_instance(context, instance_id, **updates) + instances.append(instance) # TODO(vish): This probably should be done in the scheduler # or in compute as a call. The network should be @@ -165,39 +179,6 @@ class ComputeAPI(base.Base): 'project_id': context.project_id} group = db.security_group_create(context, values) - def create_instance(self, context, security_groups=None, **kwargs): - """Creates the instance in the datastore and returns the - new instance as a mapping - - :param context: The security context - :param security_groups: list of security group ids to - attach to the instance - :param kwargs: All additional keyword args are treated - as data fields of the instance to be - created - - :retval Returns a mapping of the instance information - that has just been created - - """ - instance_ref = self.db.instance_create(context, kwargs) - inst_id = instance_ref['id'] - # Set sane defaults if not specified - if kwargs.get('display_name') is None: - display_name = "Server %s" % instance_ref['internal_id'] - instance_ref['display_name'] = display_name - self.db.instance_update(context, inst_id, - {'display_name': display_name}) - - elevated = context.elevated() - if not security_groups: - security_groups = [] - for security_group_id in security_groups: - self.db.instance_add_security_group(elevated, - inst_id, - security_group_id) - return instance_ref - def update_instance(self, context, instance_id, **kwargs): """Updates the instance in the datastore. @@ -210,7 +191,7 @@ class ComputeAPI(base.Base): :retval None """ - self.db.instance_update(context, instance_id, kwargs) + return self.db.instance_update(context, instance_id, kwargs) def delete_instance(self, context, instance_id): logging.debug("Going to try and terminate %d" % instance_id) @@ -264,6 +245,19 @@ class ComputeAPI(base.Base): else: self.db.instance_destroy(context, instance['id']) + def get_instances(self, context, project_id=None): + if project_id or not context.is_admin: + if not context.project: + return self.db.instance_get_all_by_user(context, + context.user_id) + if project_id is None: + project_id = context.project_id + return self.db.instance_get_all_by_project(context, project_id) + return self.db.instance_get_all(context) + + def get_instance(self, context, instance_id): + return self.db.instance_get_by_internal_id(context, instance_id) + def reboot(self, context, instance_id): """Reboot the given instance.""" instance = self.db.instance_get_by_internal_id(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index dd96490545..ef58f3490d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -732,6 +732,7 @@ def instance_update(context, instance_id, values): instance_ref = instance_get(context, instance_id, session=session) instance_ref.update(values) instance_ref.save(session=session) + return instance_ref def instance_add_security_group(context, instance_id, security_group_id): diff --git a/nova/flags.py b/nova/flags.py index 1f94feb08b..c6578023d9 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -259,7 +259,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.local.LocalImageService', +DEFINE_string('image_service', 'nova.image.s3.S3ImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 7c0343942b..c3f129a32d 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -67,8 +67,7 @@ def fake_wsgi(self, req): def stub_out_key_pair_funcs(stubs): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] - stubs.Set(nova.db.api, 'key_pair_get_all_by_user', - key_pair) + stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) def stub_out_image_service(stubs): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 46b9c53487..8444b6fce6 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -48,7 +48,7 @@ def return_security_group(context, instance_id, security_group_id): def instance_update(context, instance_id, kwargs): - pass + return stub_instance(instance_id) def instance_address(context, instance_id): @@ -106,11 +106,11 @@ class ServersTest(unittest.TestCase): i += 1 def test_create_instance(self): - def server_update(context, id, params): - pass - def instance_create(context, inst): - return {'id': 1, 'internal_id': 1} + return {'id': 1, 'internal_id': 1, 'display_name': ''} + + def server_update(context, id, params): + return instance_create(context, id) def fake_method(*args, **kwargs): pass diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index a55449739e..6f3ef96cbb 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -72,33 +72,27 @@ class ComputeTestCase(test.TrialTestCase): """Verify that an instance cannot be created without a display_name.""" cases = [dict(), dict(display_name=None)] for instance in cases: - ref = self.compute_api.create_instance(self.context, None, - **instance) + ref = self.compute_api.create_instances(self.context, + FLAGS.default_instance_type, None, **instance) try: - self.assertNotEqual(ref.display_name, None) + self.assertNotEqual(ref[0].display_name, None) finally: - db.instance_destroy(self.context, ref['id']) + db.instance_destroy(self.context, ref[0]['id']) def test_create_instance_associates_security_groups(self): - """Make sure create_instance associates security groups""" - inst = {} - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id + """Make sure create_instances associates security groups""" values = {'name': 'default', 'description': 'default', 'user_id': self.user.id, 'project_id': self.project.id} group = db.security_group_create(self.context, values) - ref = self.compute_api.create_instance(self.context, - security_groups=[group['id']], - **inst) - # reload to get groups - instance_ref = db.instance_get(self.context, ref['id']) + ref = self.compute_api.create_instances(self.context, + FLAGS.default_instance_type, None, security_group=['default']) try: - self.assertEqual(len(instance_ref['security_groups']), 1) + self.assertEqual(len(ref[0]['security_groups']), 1) finally: db.security_group_destroy(self.context, group['id']) - db.instance_destroy(self.context, instance_ref['id']) + db.instance_destroy(self.context, ref[0]['id']) @defer.inlineCallbacks def test_run_terminate(self): From 4f2a8c5398d4d4848f441e366e8bcc5e97a0b34f Mon Sep 17 00:00:00 2001 From: Ryan Lucio Date: Fri, 3 Dec 2010 13:50:30 -0800 Subject: [PATCH 058/119] Decreased the maximum value for instance-id generation from uint32 to int32 to avoid truncation when being entered into the instance table. Reverted fix to make internal_id column a uint --- nova/db/sqlalchemy/api.py | 2 +- nova/db/sqlalchemy/models.py | 3 +-- nova/image/local.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index dd96490545..2dc1402742 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -543,7 +543,7 @@ def instance_create(context, values): with session.begin(): while instance_ref.internal_id == None: # Instances have integer internal ids. - internal_id = random.randint(0, 2 ** 32 - 1) + internal_id = random.randint(0, 2 ** 31 - 1) if not instance_internal_id_exists(context, internal_id, session=session): instance_ref.internal_id = internal_id diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 18ba80caf6..fe0a9a9216 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -27,7 +27,6 @@ from sqlalchemy import ForeignKey, DateTime, Boolean, Text from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.schema import ForeignKeyConstraint -from sqlalchemy.databases import mysql from nova.db.sqlalchemy.session import get_session @@ -156,7 +155,7 @@ class Instance(BASE, NovaBase): """Represents a guest vm.""" __tablename__ = 'instances' id = Column(Integer, primary_key=True) - internal_id = Column(mysql.MSInteger(unsigned=True), unique=True) + internal_id = Column(Integer, unique=True) admin_pass = Column(String(255)) diff --git a/nova/image/local.py b/nova/image/local.py index 9b0cdcc50d..b445932218 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -59,7 +59,7 @@ class LocalImageService(service.BaseImageService): """ Store the image data and return the new image id. """ - id = random.randint(0, 2 ** 32 - 1) + id = random.randint(0, 2 ** 31 - 1) data['id'] = id self.update(context, id, data) return id From 1637de18a86712c52d89441c154a8e9aae6fb503 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 6 Dec 2010 12:42:34 +0000 Subject: [PATCH 059/119] pylint and pep8 fixes --- nova/virt/xenapi/network_utils.py | 13 ++++--- nova/virt/xenapi/novadeps.py | 59 ++++++++++++++++++++++--------- nova/virt/xenapi/vm_utils.py | 30 ++++++++++------ nova/virt/xenapi/vmops.py | 18 +++++++--- nova/virt/xenapi/volumeops.py | 2 ++ nova/virt/xenapi_conn.py | 15 +++++++- 6 files changed, 101 insertions(+), 36 deletions(-) diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xenapi/network_utils.py index b58b9159c2..8cb4cce3a7 100644 --- a/nova/virt/xenapi/network_utils.py +++ b/nova/virt/xenapi/network_utils.py @@ -15,20 +15,25 @@ # under the License. """ -Helper methods for operations related to the management of network records and -their attributes like bridges, PIFs, QoS, as well as their lookup functions. +Helper methods for operations related to the management of network +records and their attributes like bridges, PIFs, QoS, as well as +their lookup functions. """ from twisted.internet import defer class NetworkHelper(): - def __init__(self, session): + """ + The class that wraps the helper methods together. + """ + def __init__(self): return @classmethod @defer.inlineCallbacks - def find_network_with_bridge(self, session, bridge): + def find_network_with_bridge(cls, session, bridge): + """ Return the network on which the bridge is attached, if found """ expr = 'field "bridge" = "%s"' % bridge networks = yield session.call_xenapi('network.get_all_records_where', expr) diff --git a/nova/virt/xenapi/novadeps.py b/nova/virt/xenapi/novadeps.py index 985998486d..a68fd8e774 100644 --- a/nova/virt/xenapi/novadeps.py +++ b/nova/virt/xenapi/novadeps.py @@ -14,10 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +""" +It captures all the inner details of Nova classes and avoid their exposure +to the implementation of the XenAPI module. One benefit of this, is to avoid +sprawl of code changes +""" + from nova import db from nova import flags -from nova import process -from nova import utils from nova import context from nova.compute import power_state @@ -53,91 +57,114 @@ flags.DEFINE_float('xenapi_task_poll_interval', class Configuration(object): + """ Wraps Configuration details into common class """ def __init__(self): self._flags = flags.FLAGS @property def xenapi_connection_url(self): + """ Return the connection url """ return self._flags.xenapi_connection_url @property def xenapi_connection_username(self): + """ Return the username used for the connection """ return self._flags.xenapi_connection_username @property def xenapi_connection_password(self): + """ Return the password used for the connection """ return self._flags.xenapi_connection_password @property def xenapi_task_poll_interval(self): + """ Return the poll interval for the connection """ return self._flags.xenapi_task_poll_interval class Instance(object): + """ Wraps up instance specifics """ @classmethod - def get_name(self, instance): + def get_name(cls, instance): + """ The name of the instance """ return instance.name @classmethod - def get_type(self, instance): + def get_type(cls, instance): + """ The type of the instance """ return instance_types.INSTANCE_TYPES[instance.instance_type] @classmethod - def get_project(self, instance): + def get_project(cls, instance): + """ The project the instance belongs """ return AuthManager().get_project(instance.project_id) @classmethod - def get_project_id(self, instance): + def get_project_id(cls, instance): + """ The id of the project the instance belongs """ return instance.project_id @classmethod - def get_image_id(self, instance): + def get_image_id(cls, instance): + """ The instance's image id """ return instance.image_id @classmethod - def get_kernel_id(self, instance): + def get_kernel_id(cls, instance): + """ The instance's kernel id """ return instance.kernel_id @classmethod - def get_ramdisk_id(self, instance): + def get_ramdisk_id(cls, instance): + """ The instance's ramdisk id """ return instance.ramdisk_id @classmethod - def get_network(self, instance): + def get_network(cls, instance): + """ The network the instance is connected to """ # TODO: is ge_admin_context the right context to retrieve? return db.project_get_network(context.get_admin_context(), instance.project_id) @classmethod - def get_mac(self, instance): + def get_mac(cls, instance): + """ The instance's MAC address """ return instance.mac_address @classmethod - def get_user(self, instance): + def get_user(cls, instance): + """ The owner of the instance """ return AuthManager().get_user(instance.user_id) class Network(object): + """ Wraps up network specifics """ @classmethod - def get_bridge(self, network): + def get_bridge(cls, network): + """ the bridge for the network """ return network.bridge class Image(object): + """ Wraps up image specifics """ @classmethod - def get_url(self, image): + def get_url(cls, image): + """ the url to get the image from """ return images.image_url(image) class User(object): + """ Wraps up user specifics """ @classmethod - def get_access(self, user, project): + def get_access(cls, user, project): + """ access key """ return AuthManager().get_access_key(user, project) @classmethod - def get_secret(self, user): + def get_secret(cls, user): + """ access secret """ return user.secret diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index b68df27919..002f00c03f 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -31,12 +31,15 @@ from novadeps import User class VMHelper(): - def __init__(self, session): + """ + The class that wraps the helper methods together. + """ + def __init__(self): return @classmethod @defer.inlineCallbacks - def create_vm(self, session, instance, kernel, ramdisk): + def create_vm(cls, session, instance, kernel, ramdisk): """Create a VM record. Returns a Deferred that gives the new VM reference.""" @@ -80,7 +83,7 @@ class VMHelper(): @classmethod @defer.inlineCallbacks - def create_vbd(self, session, vm_ref, vdi_ref, userdevice, bootable): + def create_vbd(cls, session, vm_ref, vdi_ref, userdevice, bootable): """Create a VBD record. Returns a Deferred that gives the new VBD reference.""" @@ -105,7 +108,7 @@ class VMHelper(): @classmethod @defer.inlineCallbacks - def create_vif(self, session, vm_ref, network_ref, mac_address): + def create_vif(cls, session, vm_ref, network_ref, mac_address): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" @@ -127,7 +130,7 @@ class VMHelper(): @classmethod @defer.inlineCallbacks - def fetch_image(self, session, image, user, project, use_sr): + def fetch_image(cls, session, image, user, project, use_sr): """use_sr: True to put the image as a VDI in an SR, False to place it on dom0's filesystem. The former is for VM disks, the latter for its kernel and ramdisk (if external kernels are being used). @@ -135,7 +138,7 @@ class VMHelper(): url = Image.get_url(image) access = User.get_access(user, project) - logging.debug("Asking xapi to fetch %s as %s" % (url, access)) + logging.debug("Asking xapi to fetch %s as %s", url, access) fn = use_sr and 'get_vdi' or 'get_kernel' args = {} args['src_url'] = url @@ -149,11 +152,13 @@ class VMHelper(): @classmethod @utils.deferredToThread - def lookup(self, session, i): + def lookup(cls, session, i): + """ Look the instance i up, and returns it if available """ return VMHelper.lookup_blocking(session, i) @classmethod - def lookup_blocking(self, session, i): + def lookup_blocking(cls, session, i): + """ Synchronous lookup """ vms = session.get_xenapi().VM.get_by_name_label(i) n = len(vms) if n == 0: @@ -165,11 +170,13 @@ class VMHelper(): @classmethod @utils.deferredToThread - def lookup_vm_vdis(self, session, vm): + def lookup_vm_vdis(cls, session, vm): + """ Look for the VDIs that are attached to the VM """ return VMHelper.lookup_vm_vdis_blocking(session, vm) @classmethod - def lookup_vm_vdis_blocking(self, session, vm): + def lookup_vm_vdis_blocking(cls, session, vm): + """ Synchronous lookup_vm_vdis """ # Firstly we get the VBDs, then the VDIs. # TODO: do we leave the read-only devices? vbds = session.get_xenapi().VM.get_VBDs(vm) @@ -180,7 +187,8 @@ class VMHelper(): vdi = session.get_xenapi().VBD.get_VDI(vbd) # Test valid VDI record = session.get_xenapi().VDI.get_record(vdi) - except Exception, exc: + logging.debug('VDI %s is still available', record['uuid']) + except XenAPI.Failure, exc: logging.warn(exc) else: vdis.append(vdi) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d6ea5e7db7..7ea8be999b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -31,15 +31,20 @@ from network_utils import NetworkHelper class VMOps(object): + """ + Management class for VM-related tasks + """ def __init__(self, session): self._session = session def list_instances(self): + """ List VM instances """ return [self._session.get_xenapi().VM.get_name_label(vm) \ for vm in self._session.get_xenapi().VM.get_all()] @defer.inlineCallbacks def spawn(self, instance): + """ Create VM instance """ vm = yield VMHelper.lookup(self._session, Instance.get_name(instance)) if vm is not None: raise Exception('Attempted to create non-unique name %s' % @@ -71,6 +76,7 @@ class VMOps(object): @defer.inlineCallbacks def reboot(self, instance): + """ Reboot VM instance """ instance_name = Instance.get_name(instance) vm = yield VMHelper.lookup(self._session, instance_name) if vm is None: @@ -80,6 +86,7 @@ class VMOps(object): @defer.inlineCallbacks def destroy(self, instance): + """ Destroy VM instance """ vm = yield VMHelper.lookup(self._session, Instance.get_name(instance)) if vm is None: # Don't complain, just return. This lets us clean up instances @@ -91,7 +98,7 @@ class VMOps(object): task = yield self._session.call_xenapi('Async.VM.hard_shutdown', vm) yield self._session.wait_for_task(task) - except Exception, exc: + except XenAPI.Failure, exc: logging.warn(exc) # Disk clean-up if vdis: @@ -100,15 +107,16 @@ class VMOps(object): task = yield self._session.call_xenapi('Async.VDI.destroy', vdi) yield self._session.wait_for_task(task) - except Exception, exc: + except XenAPI.Failure, exc: logging.warn(exc) try: task = yield self._session.call_xenapi('Async.VM.destroy', vm) yield self._session.wait_for_task(task) - except Exception, exc: + except XenAPI.Failure, exc: logging.warn(exc) def get_info(self, instance_id): + """ Return data about VM instance """ vm = VMHelper.lookup_blocking(self._session, instance_id) if vm is None: raise Exception('instance not present %s' % instance_id) @@ -120,4 +128,6 @@ class VMOps(object): 'cpu_time': 0} def get_console_output(self, instance): - return 'FAKE CONSOLE OUTPUT' + """ Return snapshot of console """ + # TODO: implement this to fix pylint! + return 'FAKE CONSOLE OUTPUT of instance' diff --git a/nova/virt/xenapi/volumeops.py b/nova/virt/xenapi/volumeops.py index 23f79adf71..a4c7a38619 100644 --- a/nova/virt/xenapi/volumeops.py +++ b/nova/virt/xenapi/volumeops.py @@ -24,7 +24,9 @@ class VolumeOps(object): self._session = session def attach_volume(self, instance_name, device_path, mountpoint): + # FIXME: that's going to be sorted when iscsi-xenapi lands in branch return True def detach_volume(self, instance_name, mountpoint): + # FIXME: that's going to be sorted when iscsi-xenapi lands in branch return True diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 948fade7e9..e5e67128a3 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -83,47 +83,59 @@ def get_connection(_): class XenAPIConnection(object): + """ A connection to XenServer or Xen Cloud Platform """ def __init__(self, url, user, pw): session = XenAPISession(url, user, pw) self._vmops = VMOps(session) self._volumeops = VolumeOps(session) def list_instances(self): + """ List VM instances """ return self._vmops.list_instances() def spawn(self, instance): + """ Create VM instance """ self._vmops.spawn(instance) def reboot(self, instance): + """ Reboot VM instance """ self._vmops.reboot(instance) def destroy(self, instance): + """ Destroy VM instance """ self._vmops.destroy(instance) def get_info(self, instance_id): + """ Return data about VM instance """ return self._vmops.get_info(instance_id) def get_console_output(self, instance): + """ Return snapshot of console """ return self._vmops.get_console_output(instance) def attach_volume(self, instance_name, device_path, mountpoint): + """ Attach volume storage to VM instance """ return self._volumeops.attach_volume(instance_name, device_path, mountpoint) def detach_volume(self, instance_name, mountpoint): + """ Detach volume storage to VM instance """ return self._volumeops.detach_volume(instance_name, mountpoint) class XenAPISession(object): + """ The session to invoke XenAPI SDK calls """ def __init__(self, url, user, pw): self._session = XenAPI.Session(url) self._session.login_with_password(user, pw) def get_xenapi(self): + """ Return the xenapi object """ return self._session.xenapi def get_xenapi_host(self): + """ Return the xenapi host """ return self._session.xenapi.session.get_this_host(self._session.handle) @utils.deferredToThread @@ -170,12 +182,13 @@ class XenAPISession(object): error_info) deferred.errback(XenAPI.Failure(error_info)) #logging.debug('Polling task %s done.', task) - except Exception, exc: + except XenAPI.Failure, exc: logging.warn(exc) deferred.errback(exc) def _unwrap_plugin_exceptions(func, *args, **kwargs): + """ Parse exception details """ try: return func(*args, **kwargs) except XenAPI.Failure, exc: From f25a25d2693d603eb9a6f87d9629d53542219736 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 6 Dec 2010 15:53:35 +0000 Subject: [PATCH 060/119] moved XenAPI namespace definition into xenapi/__init__.py --- nova/virt/xenapi/__init__.py | 11 +++++++++++ nova/virt/xenapi/vm_utils.py | 1 + nova/virt/xenapi/vmops.py | 1 + nova/virt/xenapi_conn.py | 7 +------ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/nova/virt/xenapi/__init__.py b/nova/virt/xenapi/__init__.py index 3d598c463c..ece4304071 100644 --- a/nova/virt/xenapi/__init__.py +++ b/nova/virt/xenapi/__init__.py @@ -13,3 +13,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +""" +This is loaded late so that there's no need to install this library +when not using XenAPI +""" + +XenAPI = None +global XenAPI + +if XenAPI is None: + XenAPI = __import__('XenAPI') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 002f00c03f..52ab2901d4 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -20,6 +20,7 @@ their attributes like VDIs, VIFs, as well as their lookup functions. """ import logging +import XenAPI from twisted.internet import defer diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 7ea8be999b..3db86f1791 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -19,6 +19,7 @@ Management class for VM-related functions (spawn, reboot, etc). """ import logging +import XenAPI from twisted.internet import defer diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index e5e67128a3..2839a753c5 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -58,19 +58,14 @@ from nova import utils from xenapi.vmops import VMOps from xenapi.volumeops import VolumeOps from xenapi.novadeps import Configuration +from xenapi import XenAPI -XenAPI = None Config = Configuration() def get_connection(_): """Note that XenAPI doesn't have a read-only connection mode, so the read_only parameter is ignored.""" - # This is loaded late so that there's no need to install this - # library when not using XenAPI. - global XenAPI - if XenAPI is None: - XenAPI = __import__('XenAPI') url = Config.xenapi_connection_url username = Config.xenapi_connection_username password = Config.xenapi_connection_password From c2e328a158cadf45df9fb07f0c3da91f11ad416e Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 6 Dec 2010 19:46:42 +0000 Subject: [PATCH 061/119] fixed import module in __init__.py --- nova/virt/xenapi/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nova/virt/xenapi/__init__.py b/nova/virt/xenapi/__init__.py index ece4304071..ed8c293a34 100644 --- a/nova/virt/xenapi/__init__.py +++ b/nova/virt/xenapi/__init__.py @@ -19,8 +19,5 @@ This is loaded late so that there's no need to install this library when not using XenAPI """ -XenAPI = None -global XenAPI - -if XenAPI is None: - XenAPI = __import__('XenAPI') +XenAPI = __import__('XenAPI') +global XenAPI \ No newline at end of file From 76fd35b62bf565fe626ca30c412178894d8e579c Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 6 Dec 2010 15:14:41 -0500 Subject: [PATCH 062/119] Don't wrap HTTPAccepted in a fault. Correctly pass kwargs to update_instance. --- nova/api/openstack/servers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a9da148677..e7ab17d038 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -124,7 +124,7 @@ class Controller(wsgi.Controller): self.compute_api.delete_instance(ctxt, int(id)) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) - return faults.Fault(exc.HTTPAccepted()) + return exc.HTTPAccepted() def create(self, req): """ Creates a new server for a given user """ @@ -165,7 +165,7 @@ class Controller(wsgi.Controller): if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] - self.compute_api.update_instance(ctxt, instance['id'], update_dict) + self.compute_api.update_instance(ctxt, instance['id'], **update_dict) return exc.HTTPNoContent() def action(self, req, id): @@ -184,3 +184,4 @@ class Controller(wsgi.Controller): # TODO(gundlach): pass reboot_type, support soft reboot in # virt driver self.compute_api.reboot(ctxt, id) + return exc.HTTPAccepted() From 88c0e3e380d50d5794970063bbe464171089f260 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Tue, 7 Dec 2010 04:41:53 +0000 Subject: [PATCH 063/119] modified a few files --- nova/api/ec2/cloud.py | 1 - nova/compute/api.py | 14 ++++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index e50906ae17..a05dc0f1c4 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -760,7 +760,6 @@ class CloudController(object): ramdisk_id=kwargs.get('ramdisk_id'), name=kwargs.get('display_name'), description=kwargs.get('display_description'), - user_data=kwargs.get('user_data', ''), key_name=kwargs.get('key_name'), security_group=kwargs.get('security_group'), generate_hostname=internal_id_to_ec2_id) diff --git a/nova/compute/api.py b/nova/compute/api.py index 929342a1e9..6830bacb84 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -52,7 +52,7 @@ class ComputeAPI(base.Base): def create_instances(self, context, instance_type, image_service, image_id, network_topic, min_count=1, max_count=1, kernel_id=None, ramdisk_id=None, name='', - description='', user_data='', key_name=None, + description='', key_name=None, key_data=None, security_group='default', generate_hostname=generate_default_hostname): """Create the number of instances requested if quote and @@ -143,8 +143,8 @@ class ComputeAPI(base.Base): {"method": "setup_fixed_ip", "args": {"address": address}}) - logging.debug("Casting to scheduler for %s/%s's instance %s" % - (context.project_id, context.user_id, instance_id)) + logging.debug("Casting to scheduler for %s/%s's instance %s", + context.project_id, context.user_id, instance_id) rpc.cast(context, FLAGS.scheduler_topic, {"method": "run_instance", @@ -154,6 +154,12 @@ class ComputeAPI(base.Base): return instances def ensure_default_security_group(self, context): + """ Create security group for the security context if it + does not already exist + + :param context: the security context + + """ try: db.security_group_get_by_name(context, context.project_id, 'default') @@ -162,7 +168,7 @@ class ComputeAPI(base.Base): 'description': 'default', 'user_id': context.user_id, 'project_id': context.project_id} - group = db.security_group_create(context, values) + db.security_group_create(context, values) def create_instance(self, context, security_groups=None, **kwargs): """Creates the instance in the datastore and returns the From 09ebc4c33ff52c352cdab54fea41d1b116a446f4 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Tue, 7 Dec 2010 11:31:43 +0000 Subject: [PATCH 064/119] addressed review comments, complied with HACKING guidelines --- nova/virt/xenapi/__init__.py | 8 -- nova/virt/xenapi/novadeps.py | 170 ----------------------------------- nova/virt/xenapi/vm_utils.py | 36 +++++--- nova/virt/xenapi/vmops.py | 42 ++++----- nova/virt/xenapi_conn.py | 36 +++++--- 5 files changed, 70 insertions(+), 222 deletions(-) delete mode 100644 nova/virt/xenapi/novadeps.py diff --git a/nova/virt/xenapi/__init__.py b/nova/virt/xenapi/__init__.py index ed8c293a34..3d598c463c 100644 --- a/nova/virt/xenapi/__init__.py +++ b/nova/virt/xenapi/__init__.py @@ -13,11 +13,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -""" -This is loaded late so that there's no need to install this library -when not using XenAPI -""" - -XenAPI = __import__('XenAPI') -global XenAPI \ No newline at end of file diff --git a/nova/virt/xenapi/novadeps.py b/nova/virt/xenapi/novadeps.py deleted file mode 100644 index a68fd8e774..0000000000 --- a/nova/virt/xenapi/novadeps.py +++ /dev/null @@ -1,170 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -It captures all the inner details of Nova classes and avoid their exposure -to the implementation of the XenAPI module. One benefit of this, is to avoid -sprawl of code changes -""" - -from nova import db -from nova import flags -from nova import context - -from nova.compute import power_state -from nova.auth.manager import AuthManager -from nova.compute import instance_types -from nova.virt import images - -XENAPI_POWER_STATE = { - 'Halted': power_state.SHUTDOWN, - 'Running': power_state.RUNNING, - 'Paused': power_state.PAUSED, - 'Suspended': power_state.SHUTDOWN, # FIXME - 'Crashed': power_state.CRASHED} - - -flags.DEFINE_string('xenapi_connection_url', - None, - 'URL for connection to XenServer/Xen Cloud Platform.' - ' Required if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_username', - 'root', - 'Username for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_password', - None, - 'Password for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_float('xenapi_task_poll_interval', - 0.5, - 'The interval used for polling of remote tasks ' - '(Async.VM.start, etc). Used only if ' - 'connection_type=xenapi.') - - -class Configuration(object): - """ Wraps Configuration details into common class """ - def __init__(self): - self._flags = flags.FLAGS - - @property - def xenapi_connection_url(self): - """ Return the connection url """ - return self._flags.xenapi_connection_url - - @property - def xenapi_connection_username(self): - """ Return the username used for the connection """ - return self._flags.xenapi_connection_username - - @property - def xenapi_connection_password(self): - """ Return the password used for the connection """ - return self._flags.xenapi_connection_password - - @property - def xenapi_task_poll_interval(self): - """ Return the poll interval for the connection """ - return self._flags.xenapi_task_poll_interval - - -class Instance(object): - """ Wraps up instance specifics """ - - @classmethod - def get_name(cls, instance): - """ The name of the instance """ - return instance.name - - @classmethod - def get_type(cls, instance): - """ The type of the instance """ - return instance_types.INSTANCE_TYPES[instance.instance_type] - - @classmethod - def get_project(cls, instance): - """ The project the instance belongs """ - return AuthManager().get_project(instance.project_id) - - @classmethod - def get_project_id(cls, instance): - """ The id of the project the instance belongs """ - return instance.project_id - - @classmethod - def get_image_id(cls, instance): - """ The instance's image id """ - return instance.image_id - - @classmethod - def get_kernel_id(cls, instance): - """ The instance's kernel id """ - return instance.kernel_id - - @classmethod - def get_ramdisk_id(cls, instance): - """ The instance's ramdisk id """ - return instance.ramdisk_id - - @classmethod - def get_network(cls, instance): - """ The network the instance is connected to """ - # TODO: is ge_admin_context the right context to retrieve? - return db.project_get_network(context.get_admin_context(), - instance.project_id) - - @classmethod - def get_mac(cls, instance): - """ The instance's MAC address """ - return instance.mac_address - - @classmethod - def get_user(cls, instance): - """ The owner of the instance """ - return AuthManager().get_user(instance.user_id) - - -class Network(object): - """ Wraps up network specifics """ - - @classmethod - def get_bridge(cls, network): - """ the bridge for the network """ - return network.bridge - - -class Image(object): - """ Wraps up image specifics """ - - @classmethod - def get_url(cls, image): - """ the url to get the image from """ - return images.image_url(image) - - -class User(object): - """ Wraps up user specifics """ - - @classmethod - def get_access(cls, user, project): - """ access key """ - return AuthManager().get_access_key(user, project) - - @classmethod - def get_secret(cls, user): - """ access secret """ - return user.secret diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 52ab2901d4..407acda6e9 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -25,10 +25,17 @@ import XenAPI from twisted.internet import defer from nova import utils +from nova.auth.manager import AuthManager +from nova.compute import instance_types +from nova.virt import images +from nova.compute import power_state -from novadeps import Instance -from novadeps import Image -from novadeps import User +XENAPI_POWER_STATE = { + 'Halted': power_state.SHUTDOWN, + 'Running': power_state.RUNNING, + 'Paused': power_state.PAUSED, + 'Suspended': power_state.SHUTDOWN, # FIXME + 'Crashed': power_state.CRASHED} class VMHelper(): @@ -44,7 +51,7 @@ class VMHelper(): """Create a VM record. Returns a Deferred that gives the new VM reference.""" - instance_type = Instance.get_type(instance) + instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) rec = { @@ -76,10 +83,9 @@ class VMHelper(): 'user_version': '0', 'other_config': {}, } - logging.debug('Created VM %s...', Instance.get_name(instance)) + logging.debug('Created VM %s...', instance.name) vm_ref = yield session.call_xenapi('VM.create', rec) - logging.debug('Created VM %s as %s.', - Instance.get_name(instance), vm_ref) + logging.debug('Created VM %s as %s.', instance.name, vm_ref) defer.returnValue(vm_ref) @classmethod @@ -137,14 +143,14 @@ class VMHelper(): its kernel and ramdisk (if external kernels are being used). Returns a Deferred that gives the new VDI UUID.""" - url = Image.get_url(image) - access = User.get_access(user, project) + url = images.image_url(image) + access = AuthManager().get_access_key(user, project) logging.debug("Asking xapi to fetch %s as %s", url, access) fn = use_sr and 'get_vdi' or 'get_kernel' args = {} args['src_url'] = url args['username'] = access - args['password'] = User.get_secret(user) + args['password'] = user.secret if use_sr: args['add_partition'] = 'true' task = yield session.async_call_plugin('objectstore', fn, args) @@ -179,7 +185,7 @@ class VMHelper(): def lookup_vm_vdis_blocking(cls, session, vm): """ Synchronous lookup_vm_vdis """ # Firstly we get the VBDs, then the VDIs. - # TODO: do we leave the read-only devices? + # TODO(Armando): do we leave the read-only devices? vbds = session.get_xenapi().VM.get_VBDs(vm) vdis = [] if vbds: @@ -197,3 +203,11 @@ class VMHelper(): return vdis else: return None + + @classmethod + def compile_info(cls, record): + return {'state': XENAPI_POWER_STATE[record['power_state']], + 'max_mem': long(record['memory_static_max']) >> 10, + 'mem': long(record['memory_dynamic_max']) >> 10, + 'num_cpu': record['VCPUs_max'], + 'cpu_time': 0} diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3db86f1791..3696782b31 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -23,12 +23,11 @@ import XenAPI from twisted.internet import defer -from novadeps import XENAPI_POWER_STATE -from novadeps import Instance -from novadeps import Network - -from vm_utils import VMHelper -from network_utils import NetworkHelper +from nova import db +from nova import context +from nova.auth.manager import AuthManager +from nova.virt.xenapi.network_utils import NetworkHelper +from nova.virt.xenapi.vm_utils import VMHelper class VMOps(object): @@ -46,39 +45,40 @@ class VMOps(object): @defer.inlineCallbacks def spawn(self, instance): """ Create VM instance """ - vm = yield VMHelper.lookup(self._session, Instance.get_name(instance)) + vm = yield VMHelper.lookup(self._session, instance.name) if vm is not None: raise Exception('Attempted to create non-unique name %s' % - Instance.get_name(instance)) + instance.name) - bridge = Network.get_bridge(Instance.get_network(instance)) + bridge = db.project_get_network(context.get_admin_context(), + instance.project_id).bridge network_ref = \ yield NetworkHelper.find_network_with_bridge(self._session, bridge) - user = Instance.get_user(instance) - project = Instance.get_project(instance) + user = AuthManager().get_user(instance.user_id) + project = AuthManager().get_project(instance.project_id) vdi_uuid = yield VMHelper.fetch_image(self._session, - Instance.get_image_id(instance), user, project, True) + instance.image_id, user, project, True) kernel = yield VMHelper.fetch_image(self._session, - Instance.get_kernel_id(instance), user, project, False) + instance.kernel_id, user, project, False) ramdisk = yield VMHelper.fetch_image(self._session, - Instance.get_ramdisk_id(instance), user, project, False) + instance.ramdisk_id, user, project, False) vdi_ref = yield self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) vm_ref = yield VMHelper.create_vm(self._session, instance, kernel, ramdisk) yield VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) if network_ref: yield VMHelper.create_vif(self._session, vm_ref, - network_ref, Instance.get_mac(instance)) + network_ref, instance.mac_address) logging.debug('Starting VM %s...', vm_ref) yield self._session.call_xenapi('VM.start', vm_ref, False, False) - logging.info('Spawning VM %s created %s.', Instance.get_name(instance), + logging.info('Spawning VM %s created %s.', instance.name, vm_ref) @defer.inlineCallbacks def reboot(self, instance): """ Reboot VM instance """ - instance_name = Instance.get_name(instance) + instance_name = instance.name vm = yield VMHelper.lookup(self._session, instance_name) if vm is None: raise Exception('instance not present %s' % instance_name) @@ -88,7 +88,7 @@ class VMOps(object): @defer.inlineCallbacks def destroy(self, instance): """ Destroy VM instance """ - vm = yield VMHelper.lookup(self._session, Instance.get_name(instance)) + vm = yield VMHelper.lookup(self._session, instance.name) if vm is None: # Don't complain, just return. This lets us clean up instances # that have already disappeared from the underlying platform. @@ -122,11 +122,7 @@ class VMOps(object): if vm is None: raise Exception('instance not present %s' % instance_id) rec = self._session.get_xenapi().VM.get_record(vm) - return {'state': XENAPI_POWER_STATE[rec['power_state']], - 'max_mem': long(rec['memory_static_max']) >> 10, - 'mem': long(rec['memory_dynamic_max']) >> 10, - 'num_cpu': rec['VCPUs_max'], - 'cpu_time': 0} + return VMHelper.compile_info(rec) def get_console_output(self, instance): """ Return snapshot of console """ diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 2839a753c5..a2eac4dc2b 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -49,26 +49,42 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. import logging import xmlrpclib +import XenAPI from twisted.internet import defer from twisted.internet import reactor from nova import utils +from nova import flags +from nova.virt.xenapi.vmops import VMOps +from nova.virt.xenapi.volumeops import VolumeOps -from xenapi.vmops import VMOps -from xenapi.volumeops import VolumeOps -from xenapi.novadeps import Configuration -from xenapi import XenAPI - -Config = Configuration() +FLAGS = flags.FLAGS +flags.DEFINE_string('xenapi_connection_url', + None, + 'URL for connection to XenServer/Xen Cloud Platform.' + ' Required if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_username', + 'root', + 'Username for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_password', + None, + 'Password for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_float('xenapi_task_poll_interval', + 0.5, + 'The interval used for polling of remote tasks ' + '(Async.VM.start, etc). Used only if ' + 'connection_type=xenapi.') def get_connection(_): """Note that XenAPI doesn't have a read-only connection mode, so the read_only parameter is ignored.""" - url = Config.xenapi_connection_url - username = Config.xenapi_connection_username - password = Config.xenapi_connection_password + url = FLAGS.xenapi_connection_url + username = FLAGS.xenapi_connection_username + password = FLAGS.xenapi_connection_password if not url or password is None: raise Exception('Must specify xenapi_connection_url, ' 'xenapi_connection_username (optionally), and ' @@ -165,7 +181,7 @@ class XenAPISession(object): #logging.debug('Polling task %s...', task) status = self._session.xenapi.task.get_status(task) if status == 'pending': - reactor.callLater(Config.xenapi_task_poll_interval, + reactor.callLater(FLAGS.xenapi_task_poll_interval, self._poll_task, task, deferred) elif status == 'success': result = self._session.xenapi.task.get_result(task) From c1a40a8381ae3e559b3faad4a93ffec1abe8907f Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 7 Dec 2010 10:06:49 -0800 Subject: [PATCH 065/119] Added docstring for get_instances. --- nova/compute/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index 995bed91bc..cb23dae555 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -246,6 +246,9 @@ class ComputeAPI(base.Base): self.db.instance_destroy(context, instance['id']) def get_instances(self, context, project_id=None): + """Get all instances, possibly filtered by project ID or + user ID. If there is no filter and the context is an admin, + it will retreive all instances in the system.""" if project_id or not context.is_admin: if not context.project: return self.db.instance_get_all_by_user(context, From d7ca22cce7df319efc57a2e8224016817c92bbdb Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Tue, 7 Dec 2010 18:57:44 +0000 Subject: [PATCH 066/119] importing XenAPI module loaded late --- nova/virt/xenapi/vm_utils.py | 9 ++++++--- nova/virt/xenapi/vmops.py | 6 +++++- nova/virt/xenapi_conn.py | 8 +++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 407acda6e9..99d484ca2b 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -20,15 +20,14 @@ their attributes like VDIs, VIFs, as well as their lookup functions. """ import logging -import XenAPI from twisted.internet import defer from nova import utils from nova.auth.manager import AuthManager from nova.compute import instance_types -from nova.virt import images from nova.compute import power_state +from nova.virt import images XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, @@ -37,13 +36,17 @@ XENAPI_POWER_STATE = { 'Suspended': power_state.SHUTDOWN, # FIXME 'Crashed': power_state.CRASHED} +XenAPI = None + class VMHelper(): """ The class that wraps the helper methods together. """ def __init__(self): - return + global XenAPI + if XenAPI is None: + XenAPI = __import__('XenAPI') @classmethod @defer.inlineCallbacks diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3696782b31..d36cdaea5b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -19,7 +19,6 @@ Management class for VM-related functions (spawn, reboot, etc). """ import logging -import XenAPI from twisted.internet import defer @@ -29,12 +28,17 @@ from nova.auth.manager import AuthManager from nova.virt.xenapi.network_utils import NetworkHelper from nova.virt.xenapi.vm_utils import VMHelper +XenAPI = None + class VMOps(object): """ Management class for VM-related tasks """ def __init__(self, session): + global XenAPI + if XenAPI is None: + XenAPI = __import__('XenAPI') self._session = session def list_instances(self): diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index a2eac4dc2b..26b30bf927 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -49,7 +49,6 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. import logging import xmlrpclib -import XenAPI from twisted.internet import defer from twisted.internet import reactor @@ -78,10 +77,17 @@ flags.DEFINE_float('xenapi_task_poll_interval', '(Async.VM.start, etc). Used only if ' 'connection_type=xenapi.') +XenAPI = None + def get_connection(_): """Note that XenAPI doesn't have a read-only connection mode, so the read_only parameter is ignored.""" + # This is loaded late so that there's no need to install this + # library when not using XenAPI. + global XenAPI + if XenAPI is None: + XenAPI = __import__('XenAPI') url = FLAGS.xenapi_connection_url username = FLAGS.xenapi_connection_username password = FLAGS.xenapi_connection_password From 994f2820676b47b4f2e919d5ae7d2f9eb66c4372 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 7 Dec 2010 20:25:24 +0100 Subject: [PATCH 067/119] Add Ryan Lucio to Authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index ef1a535ca9..62f0c49d5d 100644 --- a/Authors +++ b/Authors @@ -20,6 +20,7 @@ Michael Gundlach Monty Taylor Paul Voccio Rick Clark +Ryan Lucio Soren Hansen Todd Willey Vishvananda Ishaya From 06c5889936cec1be503595915a0e0df2c4f925a8 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 19:35:05 +0000 Subject: [PATCH 068/119] Adding myself to the authors list --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index ef1a535ca9..a35398d5d3 100644 --- a/Authors +++ b/Authors @@ -25,3 +25,4 @@ Todd Willey Vishvananda Ishaya Youcef Laribi Zhixue Wu +Ryan Lane From bf34529e75022451f3833552df0e807139d0e498 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 7 Dec 2010 21:35:15 +0100 Subject: [PATCH 069/119] Make sure Authors check also works for pending merges (otherwise stuff can get merged that will make the next merge fail this check). --- nova/tests/misc_unittest.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/nova/tests/misc_unittest.py b/nova/tests/misc_unittest.py index 856060afa2..667c63ad09 100644 --- a/nova/tests/misc_unittest.py +++ b/nova/tests/misc_unittest.py @@ -15,7 +15,6 @@ # under the License. import os -import subprocess from nova import test from nova.utils import parse_mailmap, str_dict_replace @@ -24,18 +23,23 @@ from nova.utils import parse_mailmap, str_dict_replace class ProjectTestCase(test.TrialTestCase): def test_authors_up_to_date(self): if os.path.exists('../.bzr'): - log_cmd = subprocess.Popen(["bzr", "log", "-n0"], - stdout=subprocess.PIPE) - changelog = log_cmd.communicate()[0] + contributors = set() + mailmap = parse_mailmap('../.mailmap') - contributors = set() - for l in changelog.split('\n'): - l = l.strip() - if (l.startswith('author:') or l.startswith('committer:') - and not l == 'committer: Tarmac'): - email = l.split(' ')[-1] - contributors.add(str_dict_replace(email, mailmap)) + import bzrlib.workingtree + tree = bzrlib.workingtree.WorkingTree.open('..') + tree.lock_read() + parents = tree.get_parent_ids() + g = tree.branch.repository.get_graph() + for p in parents[1:]: + rev_ids = [r for r, _ in g.iter_ancestry(parents) + if r != "null:"] + revs = tree.branch.repository.get_revisions(rev_ids) + for r in revs: + for author in r.get_apparent_authors(): + email = author.split(' ')[-1] + contributors.add(str_dict_replace(email, mailmap)) authors_file = open('../Authors', 'r').read() From d03620f31aac6e8720bb6dc19860cb609af878c6 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 21:13:54 +0000 Subject: [PATCH 070/119] Reverting last change --- Authors | 1 - 1 file changed, 1 deletion(-) diff --git a/Authors b/Authors index a35398d5d3..ef1a535ca9 100644 --- a/Authors +++ b/Authors @@ -25,4 +25,3 @@ Todd Willey Vishvananda Ishaya Youcef Laribi Zhixue Wu -Ryan Lane From a107c6f69237cd7488b9ff716e370dd01b8dd8bd Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 7 Dec 2010 16:06:55 -0600 Subject: [PATCH 071/119] Initial diagnostics import -- needs testing and cleanup. --- nova/virt/xenapi/vm_utils.py | 15 +++++++++++++++ nova/virt/xenapi/vmops.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 99d484ca2b..801867bd41 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -20,15 +20,20 @@ their attributes like VDIs, VIFs, as well as their lookup functions. """ import logging +import urllib from twisted.internet import defer +from nova import flags from nova import utils + from nova.auth.manager import AuthManager from nova.compute import instance_types from nova.compute import power_state from nova.virt import images +FLAGS = flags.FLAGS + XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, 'Running': power_state.RUNNING, @@ -214,3 +219,13 @@ class VMHelper(): 'mem': long(record['memory_dynamic_max']) >> 10, 'num_cpu': record['VCPUs_max'], 'cpu_time': 0} + + +def get_rrd(host, uuid): + """Return the VM RRD XML as a string""" + xml = urllib.urlopen("http://%s:%s@%s/vm_rrd?uuid=%s" % ( + FLAGS.xenapi_connection_username, + FLAGS.xenapi_connection_password, + host, + uuid)) + return xml.read() diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d36cdaea5b..ba73079ec7 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -21,9 +21,11 @@ Management class for VM-related functions (spawn, reboot, etc). import logging from twisted.internet import defer +from xml.dom.minidom import parseString from nova import db from nova import context + from nova.auth.manager import AuthManager from nova.virt.xenapi.network_utils import NetworkHelper from nova.virt.xenapi.vm_utils import VMHelper @@ -128,6 +130,37 @@ class VMOps(object): rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) + def get_diagnostics(self, instance_id): + """Return data about the VM diagnostics""" + vm = VMHelper.lookup_blocking(self._session, instance_id) + if vm is None: + raise Exception("instance not present %s" % instance_id) + rec = self._session.get_xenapi().VM.get_record(vm) + try: + metrics = self._session.get_xenapi().VM_guest_metrics.get_record( + rec["guest_metrics"]) + diags = { + "Power State": rec["power_state"], + "Dom ID": rec["domid"], + "UUID": rec["uuid"], + "Kernel": metrics["os_version"]["uname"], + "Distro": metrics["os_version"]["name"]} + + xml = get_rrd(self._session.get_xenapi_host()["address"], + rec["uuid"]) + rrd = parseString(xml) + for i, node in enumerate(rrd.firstChild.childNodes): + # We don't want all of the extra garbage + if i >= 3 and i <= 11: + ref = node.childNodes + # Name and Value + diags[ref[0].firstChild.data] = ref[6].firstChild.data + + return {rec["name_label"]: diags} + except XenAPI.Failure as e: + return { + rec["name_label"]: "Unable to retrieve diagnostics: %s" % e} + def get_console_output(self, instance): """ Return snapshot of console """ # TODO: implement this to fix pylint! From d647d6b070e0a910a9f20cfc1106027c86858f30 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Tue, 7 Dec 2010 16:48:42 -0600 Subject: [PATCH 072/119] Added livecd instructions plus fixed references to .conf files --- doc/source/adminguide/managing.networks.rst | 2 +- doc/source/adminguide/multi.node.install.rst | 31 +++--------- doc/source/adminguide/single.node.install.rst | 4 +- doc/source/cloud101.rst | 9 +++- doc/source/images/novascreens.png | Bin 0 -> 27949 bytes doc/source/images/novashvirtually.png | Bin 0 -> 39000 bytes doc/source/index.rst | 2 - doc/source/livecd.rst | 46 ++++++++++++++++++ 8 files changed, 64 insertions(+), 30 deletions(-) create mode 100644 doc/source/images/novascreens.png create mode 100644 doc/source/images/novashvirtually.png diff --git a/doc/source/adminguide/managing.networks.rst b/doc/source/adminguide/managing.networks.rst index b8563637e9..38c1cba78e 100644 --- a/doc/source/adminguide/managing.networks.rst +++ b/doc/source/adminguide/managing.networks.rst @@ -23,7 +23,7 @@ In Nova, users organize their cloud resources in projects. A Nova project consis Nova Network Strategies ----------------------- -Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can c-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. +Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can co-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. Read more about Nova network strategies here: diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst index 1eed30c5b3..3b06d7d911 100644 --- a/doc/source/adminguide/multi.node.install.rst +++ b/doc/source/adminguide/multi.node.install.rst @@ -35,7 +35,6 @@ Requirements for a multi-node installation * For a recommended HA setup, consider a MySQL master/slave replication, with as many slaves as you like, and probably a heartbeat to kick one of the slaves into being a master if it dies. * For performance optimization, split reads and writes to the database. MySQL proxy is the easiest way to make this work if running MySQL. - Assumptions ^^^^^^^^^^^ @@ -69,14 +68,14 @@ Step 1 Use apt-get to get the latest code It is highly likely that there will be errors when the nova services come up since they are not yet configured. Don't worry, you're only at step 1! -Step 2 Setup configuration files (installed in /etc/nova) +Step 2 Setup configuration file (installed in /etc/nova) --------------------------------------------------------- Note: CC_ADDR= -1. These need to be defined in EACH configuration file +Nova development has consolidated all .conf files to nova.conf as of November 2010. References to specific .conf files may be ignored. -:: +#. These need to be defined in the nova.conf configuration file:: --sql_connection=mysql://root:nova@$CC_ADDR/nova # location of nova sql db --s3_host=$CC_ADDR # This is where nova is hosting the objectstore service, which @@ -87,31 +86,17 @@ Note: CC_ADDR= --ec2_url=http://$CC_ADDR:8773/services/Cloud --network_manager=nova.network.manager.FlatManager # simple, no-vlan networking type - -2. nova-manage specific flags - -:: + --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 + --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 - -3. nova-network specific flags - -:: - - --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 - --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 - -4. Create a nova group - -:: +#. Create a nova group:: sudo addgroup nova -5. nova-objectstore specific flags < no specific config needed > - -Config files should be have their owner set to root:nova, and mode set to 0640, since they contain your MySQL server's root password. +The Nova config file should have its owner set to root:nova, and mode set to 0640, since they contain your MySQL server's root password. :: @@ -121,7 +106,7 @@ Config files should be have their owner set to root:nova, and mode set to 0640, Step 3 Setup the sql db ----------------------- -1. First you 'preseed' (using vishy's :doc:`../quickstart`). Run this as root. +1. First you 'preseed' (using the Quick Start method :doc:`../quickstart`). Run this as root. :: diff --git a/doc/source/adminguide/single.node.install.rst b/doc/source/adminguide/single.node.install.rst index f6b2290bcf..8572c5a4a5 100644 --- a/doc/source/adminguide/single.node.install.rst +++ b/doc/source/adminguide/single.node.install.rst @@ -9,7 +9,7 @@ The fastest way to get a test cloud running is through our :doc:`../quickstart`. Step 1 and 2: Get the latest Nova code system software ------------------------------------------------------ -Depending on your system, the mehod for accomplishing this varies +Depending on your system, the method for accomplishing this varies .. toctree:: :maxdepth: 1 @@ -139,7 +139,7 @@ Type or copy/paste the following to source the novarc file in your current worki Step 9: Pat yourself on the back :) ----------------------------------- -Congratulations, your cloud is up and running, you’ve created an admin user, retrieved the user's credentials and put them in your environment. +Congratulations, your cloud is up and running, you’ve created an admin user, created a network, retrieved the user's credentials and put them in your environment. Now you need an image. diff --git a/doc/source/cloud101.rst b/doc/source/cloud101.rst index 87db5af1eb..7c79d2a70f 100644 --- a/doc/source/cloud101.rst +++ b/doc/source/cloud101.rst @@ -54,6 +54,8 @@ Cloud computing offers different service models depending on the capabilities a The US-based National Institute of Standards and Technology offers definitions for cloud computing and the service models that are emerging. +These definitions are summarized from http://csrc.nist.gov/groups/SNS/cloud-computing/. + SaaS - Software as a Service ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -72,12 +74,15 @@ IaaS - Infrastructure as a Service Provides infrastructure such as computer instances, network connections, and storage so that people can run any software or operating system. -.. todo:: Use definitions from http://csrc.nist.gov/groups/SNS/cloud-computing/ and attribute NIST Types of Cloud Deployments -------------------------- -.. todo:: describe public/private/hybrid/etc +When you hear terms such as public cloud or private cloud, these refer to the deployment model for the cloud. A private cloud operates for a single organization, but can be managed on-premise or off-premise. A public cloud has an infrastructure that is available to the general public or a large industry group and is likely owned by a cloud services company. + +The NIST also defines community cloud as shared by several organizations supporting a specific community with shared concerns. + +A hybrid cloud can be a deployment model, as a composition of both public and private clouds, or a hybrid model for cloud computing may involve both virtual and physical servers. Work in the Clouds ------------------ diff --git a/doc/source/images/novascreens.png b/doc/source/images/novascreens.png new file mode 100644 index 0000000000000000000000000000000000000000..0fe3279cf89b2290e6365b784792df692c37595a GIT binary patch literal 27949 zcmeFZXIPV6(=H51N17r?6BHE@K`GL!DCmv!UL(>=C_(^%&{RMWP!Ui%gkGg3)Sw_; zdI=DEhY)%QBm{PR-beBMz90M8|L;FJj;vg)HEZUanRDh^i?`aEYV@=mv?L@X^iLi? z)FmMy>n9;0ZJ?$kj$q$@3@28rKviVL&?L%7gJt+{KQLr%|u&?{6j`QIVrO?3lj~Q>h*|^`u9jfw@sVs zH^GY2c$v14C+T8%uNRNCIB>*-~UXg|wfZ;VnHM{!n$>q2LYfrAseTYv8NnAM9IkHYa4cL@K zHF;Bs$m6vvp3#wX*^IfGpX&MS%8^)HxYrkbT5$0;8Oavaw+Ipe(L3wuJK90Q4rA+% zr01T!0jBdrkkIhbv4n8+li|LR8~9EK|S#zU0Jr07MJB%~OHLq)YIoRs7p z$Y4sctCWo2`0kOleHZtoU;Zv?OqD##KlX~3nr`muwI8$|DxvSCuG0yqaJ*Nrp)^s6 zc`v(ojzal^HrXdyTNMgzb}tI2VAlt!{pVI`#7P@pvt}5q3hvW|Jk;j9%#=cX^)<@_ z_RQ#ekH)TP@&+(vMo2y6xW=Xvny)&kH2FwQ=^bw_^Iin#2jg|6`S(vBTV|8%vKK^E zDal-I{9x5_-G;S-+9cXV)%$Ard%cdAi%Ma@p_Y&_3eOCBMZb!3sE~>TSd29K)F2}TMiRH2^QnaqL zCbZ{h`6GEFSt74RqF5SP;#hbD4sNa8q7X0;5V|EVU@fqB%UPiQR=SR`-lh(7{_BF_ z0+)j0d`-Q0?fZK2*?uY*`tX*WIem9SDbY=qQgTG3|2wm9Xx$;bC>@k;y|$X}iY@@M z2?=~cppR{*o2%Z|Jfsc~hLb7=r?+hNGdW&&eCH_V_|dV;adYtLU_fq8_NDBRTu@Fl zL?{ag0jt(4t3Ba=n5Vv?ocPd46Y)e+9s5e@dI!@T>X6r5&7kIs&9w3fmJyce({T#X z@UXz(z_$_CcQ1n8UVnbOJmK(t&Cg&fCcArf+twX+4$m)JkJ)6~f3mN#e`j^ae#0uo z{(&fs2)h`?y(N*U=Sfz$)}NilJsYhlos&Q1IvaYLd$U^STR*lAHjDP`M#YC4q>rTE zPp?kJ2ysSgyz7Y+N-Ft?zKW!jJy*{m`&K@Rz={`SAP>JBeqT*ajlTU>JENwsrb;hU zuWm1CFTLfhrPupEqhu@YJ&S~PvUeK+q?l`*?r-?}s{ zGA*xQr63DufwRJ4$NI-z$J57ejtR%fWRqm=}C@g7oX>ZfvXj$nPXk6)N=%lGr znVr~7neCV*SYs}_vIO#lvJJ4%Tq#L73A53VRXn=2Ty;Xfn@#N+cu}+pw@6JA5GF?sGBV83;szKU8(!uJ% zGTT+#r#8AabvE#>$j*;8-L~o7b4%wJKQ75ELKjXGs-G!d^fSYyAO&)4Oi_{r`B*$yQc+Zx&yne;j)_Woozye|s*Z;=t2M@q zUa`hs>3Fm{?oJ84amantJL)0lM{yG*J*}J~aJ+9^EH1C}Sn#;{^p7 zfG&dY`|th2{7n3;ar-!LoGs3iGJsNnGMX|uj5cg146N$Frw0jum{@CCfAn~@ll%7L zldR4Rf$D;2$1iJHUp||BHbIE|5Q(#9i)j>L6K524gORYdv9jHnHl;NSGo7w%Ha0Ph zHv@l7DA)XYrQ+&D&A2)2$g#*e*&aWk`9pZHbS7WOe{o!+Ug{8|J$NOmw_BK2m|a+m|FzbS6yN!-Wscd21y$7Y+~{(Qw~u?%CVb0k zP6xfSkiF9x8JB$BLRZ9bO0T=J-*ZHLs^}XE0;}^0!A8c`^--i705oQ<&Cpi}*S>35 zvR^b>f}!2!f_hHxeY=`*HZMQ+NaP@TLlaS7LF;7d4maq^h2Ct z4x-JXc76G&1u{`G>I$+60*NL`FW)bHxt}7GYVoNtU8Rd%f#0)qanh>PXVWd`SHK93l?ZT5nivmAy9|sPg=i{y&WfeXzobQ-Fx={HsMmJ<|F9&Z6 zw;fuNSrWR$b?M6`WTM<%4MQgb2_p|fTO&f2WLG_`v&B8i;BIkmJp9Xi&g&rf*&!!b-%FZ^Z z2@Pzr80@JV1||TzKf~H!;Sui3j}rEhJlM3Wcf$vSTem;Vdtx7yXo^Vn|A;LG?Rh6+ z?;R{+bTPS@E7P&{r;xkZZVMrOuW|xmCx~8Fl`I>cC);!;=Z6JE9+rBO7#8F-Z3 zicrta!C>dMu!%zovx+R>#gnnG*XjowoxaMzmm8oBS1RDJ4?AzS3+5|3R*--y_&e7t zGv?E@NSWiLy=V-wdTVzEA7p&;8P9&`=_czO4$2EOdg-&8*ishjKY$-2e4NQ@s%Lo0 zIL$?QzBvT6z{j$=g{=u5YFx?v(zLWTqK3?p!xPD`PuCY3{6b;pyAguK|gL} z3;x4Y9k$u>>6yx{VgXD6i-+#;M_&&PTT?w5p9j|okDl|lkX^iADWQJZ`65y#{8k>1 zksy|#?8?l`lSj(znPSauk1qY_jdFXusy_OdyQ8MVyR%Z@$1P34?t;ZaSFZc|O~pIv z-KO}$-qN&!z%Sn;KQwqWJSk)MdfF}5n>F&HGwB(`73G-d8Z@4XJ_NUu<|(XbCMH}2 zm0ogPnbMPcpjG$rr(Ar)VZ+WeQh^TMvS=1)dm=!(sMO8-=GpQr`h7uv+t)?+DC*6K zyB`{LY6pZx=0%#Ghc9%^chYpd{rsaKyKpgBPbQB-_p+|z(CQ!!yO;g8y?~g`mvEE% zae(J&pU^VAD_KtQ;|!qxWoGTy<>!t%t{zTL0vugT{(1SNHC25fKFWYIQL@a%&}}rJ zfDnMuo5EGcU=K=7N@JJpq(*OuIO`X6AL;m=Y($TW!#-o`RRByt=gN|0hO_^gJ z`z>1QQ%MT>2%LMawKRw zrShHgC;WS(FLy(mOrB{xHXk#}w)Cqk)_$$$N;CH2_A2u1+uoKdcr?!o^w;#g@0W+K!QK9_)*%>(B}APICBwWL z3ss<)VeF&_(Js)L(Yr-Q#OOyKbB}PQazQRdMS{6}gr`MAKHp0GEz1KP;V+<0^Sg)fYwJGuQPr}b4rg&pF+CX~hckBio59z6Xi83(SM( zU$liE$1VvP8JQvKMyVvy0{sXt?e%Pn4w8ErC;PT!1)FYSix|01TNSCi7_Nq1R;J7v z)8SiXc}HnSg;enf8~pGOgW4;z%)n4LNI|x|TH-$ukl1T&qCZ4$irU^=DK~!(wBoc0 z`o8mpu21M2qy^QR*ahI*)^<1etol%OSo4l$!+J-?k3tq@(ULfIyS0%Rto9rOOc>Tt2rw4_9XxPVt0 z*@l>$dBP){-}-1O@mJlH;m?dSjkC}aNlB8j_%t71n4%g7O_`3<@ohUT_P*)4oL5XW z{ziIw<J zdoTB?Hs%LPA2?y7znLZ1Hu&t(E#64LDk@|d#Ys>?g*T{==T3RJ=VG&Dt#B$4`>rOX6Zc(yYkMON}BaWIBSK29wYlnyt>hi{p7r3VKGHGRLJr z!xAjx&iFLgKnh^@p4@fx-6eY72ReERO#;GB|=A_}8k|Ca9mkmhrQ~v1W02eEJ?e zC%YhhQzVypnTgddfpk<`LIKKt;K`+$a=BN{++@;Fj!Q1xm+Hvru##e){M;M4H&YMI zA5FD3{5Ti;;~$%U4F3T=>FX-)3{3F9dunc_v{R~ExeJP?96>pisL2jf-Om+Ito8;u zfF5Da?}N`*(7SOoN_f&~KA9rLVs=<)Ssn!CLs!i?rSuELx2 zS2(Y0YH8f1+EPldgu^=0kK@Ti&&^N>hNUnP&f{TbI`0OUMOr+5?v&DC-iCL=^H_D| zbZ2s8?VhHCeoppTcJ}l%>30~~=w8q@)E&@~_I{a(TQNv-&Z5ta7SXUMF^3gyMZk=_ zuXxw*Py2rpdyjF)mTzqj>mAvTUsUgl?RnSe71Ste)R57zzo|0OH7zsK_I+hSBc!Cz zt$3y=$jK#WQfPc=JYAL=Umv9ObvgLl!*4VXLoVDSVF*57M0+&Lu}UE}OZSk%nOx>Q z=O=Q52aWx5KO32v$(ciMJ=njdz`ORo@bO(6g1Iw#kUW_SV zb!)O`(~8e1%BRQtHe}wgJi7jf@kZF-$K4A{9PzgeBt~RYNLOn zs2<7buC*t=W4C*>olM<9T|uWqjiw1=I})zD8>85KH}mS6aJ;~jK;_N$t8NAFOQZ^) zYI|r486@k5i@z49uWG37TNhu`MDd-sp>K>9X75W9{W8uYD4Jf8T`pRs30D$n1>t{Ae6qknE(L&KnI_47k|IcalPy84?>Rt&;U1uX%|tk`D`+dhWMi zJpby#l^Z+!_Nfz3cGSViu3tpl4FxI%lJDl-Nt7gvr^OXc=C^l!yB`lPTM1= zP1B^Ja0FZ;z#(8@za|H7XODa(+cz=eT{?qA=d3TR8F9;<+&b_$a!0Hlj-htg;vjwX z_O1;z`?pVfzSJ%rjkT^~MR?|oo{ws}+yfWln;V`?7aqwFfOx7sty*Ar>h{ziKnq*m zRKnmy%nX#Xq+-X|ZW3-|z%Iqx45rcdVc)Jvy_U*^^wWIC3-&EB32Zb6b6s7S=kGa(wBP83g{f~#tbCz;TG8VFm-hiT%{UooEEh9siyM1E?LWw0! z+>(UzwPX=lIy_}N?}}}yjq8P2_A=V^%ZgQ z`tV%w{NReyCUllrB@0u$lYm|0OUCt3z9)l)YK4o2RX^m4ntUf5egAFv1(}bJJE`(K z^-Kl}#jf}6+f3M@7sXd7Q`x%H>t=~n`mKsXHAk=Uz(=Y#R1BsrRj(IJe3L13#?Y%G zjNBH*yI~DYFLQ)uV=QF>{Dscd{#mfRvVtbRuNyjZ3&(Q}0}bUHJ3pQvRtAR@?c5sA z2MFPoTP8|`MK)T^zZRNG^V5trT-M0(|}s2920QY8rq zFUgaK%KE;f=qaicZiA_|9eXp0wV7 zl<0XQsLO&gnBs4eKTv=0z>lvH?FwyLQB1CUdeakj-A7C|dFHUT*2k_l#dBG<8#h_G z(%Zu>@8{JM!OSP_G5?VJ<>H44+eC{Wp(JF~yuttds3zN6=|g-H(A(dr>|1}usG!|j zke6n#eV^??Av|l0>Y-42zA+TfJhZ8~vS!^L5sT3*u;LX>C*{rKP3Wi=s!R!8orHb4Spq zcWcbn03>|!cmr5EgssfYH62a+1j#)RTcG)R?vFUn@-B58twLhg>H8(jEes8Fr9GEd zQ+=j&!Wp;$h5-Hpi3`L;oNZJEy;nCekr1PzRHsw zRYNMj6n)*6bDM6My%;VLiYziJn;pnePvli9v#2+8Ws$uc57oLcvAQ$+{gr`@l~l_- zD&@X&0_Ua8oHY3;cuJt!^p96bS226rcxy^nWvNj;!>BDjSg>mC^&$5hr8nmQv& zM%t{$IsP)SR=@{jUa)1;0J&_%mD*O>k-L}t);H_#BY{`&Z@c%u5d2pOD`~@3phDTE%~x$s>5XcI zEeu_0MRxEE{$;PB@%F7!boAa*k^Op1JrBJm?uUBW$lN)x)1ZeB!wiTR%4nq#znlRa z_WmA9QSow6qNF*roWhjuJ9G@?I#ox31nN(8nbtyqNIN-`qM>Hj893O&ca!~7Pj`>r z%SSoM4pyM$0`9J+TO@IaNKtE@9>NFcvnLess8IJnA5JrWHM`tq#WA2MTW(vACVv0M_7ZT*i zl@IqS5z;j%s1?gA0FSem_WlgAiYhB(Hk;bYZ=RJ_``sn&CN00qx| zzC+P$Z1qWdn=!(I(eVSzjq{vjg zPnqCFuvJ`{kQXun+Vcf^G-xnlAiOae_QD^CYz9q;U`A?P#w+S%O423X{BZ4t6%
}e#fYLsYShu0K+09Y&Cy3AOF6fTCa0y?hd8k&^msr)GTiXO`2s}E%&Wl&dB%Ys z=kV)e3k)MoTXm|jJ!U!2ji4$|FO5$RV>8NDp)G||Rdp3;hW`8Uwzo^~BoY!}+fRdg z4@R;S$eG4IkR8!a;m4P+BE9H&{t+6RXyLIh6Z(t;gmKGydkx(f9Yx2=vU7$&Nr0dU zYN)(<$>srXH8vB9P#`g5FY@e3x{nz(nP1j0Q$*G%uHY0*k@u5N`-Y(7&oQR{2O;fY zICidYip2dtpqS)b{1`j)Gch5)=5)45el+{g4k=J=8(N}J)hy`h!wDRHlO{YDB=x+lG zq8k|o#@nN5R6Ad(5)?Om;t73$un9oVqC%rp6TEN=FtCA5o~9n@Z3 zU2-F61f_^+17xgYpG7`eD#T*i(~s5s1NQ==J;COf-XT9POP*9wsOV~V>XPflKExNb z*bf>R4i2Z0r}BkVc9?>qrp(4R?Rz9S>T=q-4P^TIod3Gm?21SD1r8KAkz z=GoZP_v+OV6%nrv-^u7T7b~B+K%lI0=(&~x#_YiB->Dn~Q2>O`w4pFZ7X(E4eS%`9 zZJ2#CZ)72s9(#2XLHweSfFNqWg2HL$ni{^wPz&H+pS$4$g%gQk)d||(YvpcG-W4N2 zt`&FT{nL1WQ`b*0s1=h~0N!~4>_D^X_j~>%l{wsz6yNHA>=(%6^D|y?zUlQy&onMq zYCMkc2F2Vq{5cHRN-izWv&BDE10lPn+rGD zS&!81x%0W~Lo(jj06vkrz%q5B3jogx&S+zo?yezX74os|7I}^SOGV!aH`L7u7N~W% z{NvEm=`ws;(#mvpfSErwaT>hL=x5Wa8~xKUjM-n-ij^22d9Q|0s%3y34)|cKnrM)3 zxg(c?IGzyrTE@J$>3|dwZax!Mf_&a?3O*T!A|Y^@5{4NyK)PG|`gw#s=5QA?;@Do! z8>}QlK3Vpz?zETSQTp;kWHt8O;jM(eXVmPOoAGE1%T2W`LGI6OcL>rGnTo5v^ zFkJ|%lLgd#b%NB_qWzs`zj6p|O{#$>^(#3fPirfldzK^zNeZ+_2g@eC>N|xh)cP~V z8^t@;6rs5hX#`6X5&xv*p@6Rn(pD)GwS7KbW2noinCU&7UP1S~Ytxo>YAI)YwZ6GE zJ~VX`tB>RmdU?E3x7$~$wOM)fTnsKH(6{TcL3~YtBS<~4KaKl2WKg-{0DL}ciyAyf z#pY<^kCg@WXL6j3p$dy{`?MQd`6EBPY)ZDgh(Wpr%P zN-2^A)Q*FS53sfVuP`|=NyKez=MSr) zJIz-|5x@`xYf;OCS^%sfx+qYath9qvI6pT~S5rFWjYDar%4v`&NZrHM&{BczJs@6y&Xm_P=M7gD|sI6y2 z2zJkGA+1|7e59vIusAhC26VB?j!#olWqlZM%lK9NS}bO6e3J_3!?J?}`^)hWp-FLv2f1|>?>2FIP;g6- z9-4a6!-MK4Rrd1YxBp&coNS&X)Q2vH2-iFcuU+FTBVg40Hef-B4Ch|Mrht0?faplU z2YflDArt+zFHc;7uUunZd~qJyl+l;1Tc@qXNUV$5)HtQ3KrY$8Gi1brOd1WNWi2Zemu{IgKHRDUs-TwT z5FAGbui|ETe!24-GM>EI^Qkc8Pm_ph(COWg|iAuWj-l+D&Lg-bs z_2w2ThtVPIdd{2HWV*Vf1kIs$H;Sq@UJ2CAAx_!!zB5dGno(0rkuz6MZ!}^&-Mil? zvE*UlgH%yo`2O@g_b`2uGO;x9ey#M1ubHH z;Fa{GW>HExhIcD0lG6>3X%-?pR{| z?}?Lf7nb?yy$#clGYSW9UnCINPW!()YwDyAggb6EO_p@}Ock!DOkp~;xFGF3LGV(o z`!AW}dcfXV-=)Q?nq>M?JvyqUcZCD?7J>&A+ zGaoqI#tF3jM)?bKH^`(b?fbl&vc&G5DD-)wYUw#8$ghs%>KQ+-KINQ?qO94#^t9cq?j`_VAB&Wp8 za+{nsQl{`$(;xx6l`HbvKLzD_Iy$X^c!70;J~KDg&o3OLcQt*5@7iGCj^9!^Hd+q* zyhj)HNlz#1W5tBo!V*+KH(bt#(FT^3sF$g6>E&iZ(xbxjF(9Eu7MI%1W~&eNOLl<< z1rndes9|(JfD;AQR?18cf>A50S~E>*@p*tOt&J<5>BogjxE6Y>141i3??fJ72!MHi zz?sKOA8i_XqbFS!oZ?-^^1CJ_*88Ukm_CqbQ$P6SC%VF;qnECck*Vo;xqpUjJk!y~ z>QJj770h@z1s&2yn-f9hW35|~w)4*QutyQJiGUYfrEsoxU!!e2omE*sX?1f{tiQxd zM=a&u>xOV)M-XPmK^psV#|~fL*wVpS{eXq9DfG;ao#~2M-3bw8JQ^{h_NW0FbTTP0 znX%*K-0K7ME2L@OmVJ#T%2N-=j_`qA{S^{7lAm}^qK7x8x90$U2lZW7B<>;z91|}* zkigf{em@i5h!)2S4O%ER7+D-hKPAW*tTqP03r8pxFiTs36FqNB3Gb&^)eOgt;uLeg z@Z8ZY8*0iLYDXwu+FY;(cA18q1erU8H%#2y+ro9e;#vRJ;;pCLX|D05PMZRsR^+V7 zSz3|fwiYIPnsxHBe*L|A@6TzY;;r{$5N0Bxo~ehVFe)7K;)2-C65O5VzM)oF!s*7- z#n1zvw>M(d)l=b3#K=xUrg5R8WFYJYow)rDEjAJQPxY3(v9%9RCiD_#s96KhAd!=M zb5^y+Q|M=X+0OJcl|=!w7lr>+xu3`hnQc;Ii;bAV-N~Y9`z^a3lnI@TyI~O-_FvT% zT&Z+|1xk~vU-Oh}QoU|c!C@-W(^}IqgceH+{^Y4Np1;vE>nB zcgbN}q2Hs#o;G9P8;6feF3_<+Oz(6&hFJK!2x#Y2Xyj~K^2qi*$My+>7=N+1jB2{Y zrtN0B5(1aBBJun0wn@d!XF>K0WzHrG+1LIIhl~pM&$1O%d6V=q5=H#)!y&oUi&|-N z(zg^H2ea!I=Ak2Kmt{5&U^~!x*;QNqPmAXp1Jkzj-O>Vgg=)fF#Q@l&T@nGUV_tO+ zPbwr&Q&8sJz*vI!wmP$1F?4oAn#>Ksg-^j&h$FZ=yZ zM`)O~>QhF_n%{6>>N5v7a|aCaxbq zlZ0sMmWmYHghbO6u6%@r0m-hAA(9I074;zO5cc@z5wL0=cp6YXZQ-|NNXaYszLwPf zTu}~c0L&y_L1VwJ$!$IJi&(C&wpR)3^Ke1WCJAJX-NZP7q2Qw8b-c2 zGcUCiX|DUW^&iL(^TI`P>VA!HS-1Xdl7x)hA&~O7PKerZ2qht{qcLVVv#U}tNl)Kz z*!_}8gNav3k~Avoua%fGj@bV=64AwI$jDc3KGR`2Q#UD@xZ=6L{z6?a#D95^|H8DI+=fkCWJ$JwjB%?#4M7Xt)Uqe)p8G{2Q0odIo@_!|u~Zk6@`3D>3>`z4nqK&+s%?XL>w zzw4O7g8N?0Ri3wRDW+O=S84dmgLq5G6ds4SRHmwr=DKHJ3JW}om%RNfq^cRoS&vNq zQle!r^J?me8VSkSSne;-{qLt#zbz}_U-r+%-Bb#{?;HJF14MT^N7f@sHNlTrQ?M)_s90eh53phP`daAum9TYegW!mneI-lr1>kGjX9VFzaHzf#+8 z6|G;SUirzYNHsw<>z;>R%tAs?R>8P~H{CnGx|qKoOXqI6Fn1?ugqV-`kb5$vLr6WI z_P_kG-#EnG7;1edd7{OwZ=Gwk&ASl(LP5@q3e1lZ>?1Y;e@{g2ovjo_*Dt%f=ty@q zf5_Kho}ct?QSh5eYLAT7e#^!*RzSf)J77X!vdV2slDWiZ9?OkFv|tW_b48rfTQIfm z*7zOUmw#I|iAUpl%GFAa44>;T#q#aWjgEuA7Rmt8*>oj8opYE%q*ki@S*D{x>9$%1 zp`O(3`>m?P*|W%>QA?rAo{`M?2KQG?=(zrkCFZH9%l{J7WmLs$`Bey0hR-?=o#vH4 zTgxaQGf8b`PV7&%4t!CQnmfD|>)a)xK~XJHp%VmMcXT=xi2aR{jSVsdu`urDBVlKv zJ6$cBR4mqMlAjDggSiz#&?SP&?e-}1N&_v4zUaE&3JpnB{0Js#7;Z-39t`xQ-=aJ7Y0rbth!d1^P>Mco0@9)^kATFW?BW`8)+9H)`D9A1I#3a4&QpN}O5E z`zC36T=NQ7ubN46^Bm_keHiBsJ_$8GQQ?g)7eeC{B%wjbK~-}eSWa5o!9b#OespeH zeOclknKG(rAMWXi+uPk&!rx;BR^=>g=AW3#o{)X+jB$d&q24w;rno**!XJZk@b%yE znR9sBBJ0dfSFBKlljM>uqFDLku2te8ZHtw~6st9HjIq!*wA))G$IuE;;G%qP`BfP| zbI&An2Q+Cw~LaM3_O4Lp0D2Kya%39V)hAV8nm5(3kHBaBERnR;}+;dAdkRg+9zxJu~6UyLb* z{ZJU<|Kmm>30L0>$V#ACViG*o_sM=sLFBj_G*#=&4%WL{TykG{q7P92q4EC>r2Q-* zhCCU`p~`1*jg*|@!ZTVml?NMtp;>^!<)(_4#4BYYn!9+YG|vLTGc$?`nKyjGnOiQN zF+I=hWmkZNxdo};a%J40760T|$(jLHncKb$E4gvawn$4Bd1Tie?e_<;3D=ksSF(^5 z5^q{UJ*DJeEabRfZ{Gf!<-Q$IKNNF4InW<9Z^wP&DOH@<3F~@|MV1H3n``+d2LXY0 zx2^F3wKby9nS>hAIka+S&*^IvjoZroKXWK`GgI3`i#6u9^Hy3Q07l7W*a&tbmXU{T ziq!&!%R(OenVV@8AQ20(Z%e~bZtHMdS2$|vY=Unw7q{7kCbRlNp+YEP_UkI~@P7?^ zBRe>TKzcTCqwq}O17iv{Mds?OD$w3HXL}pH#nh9QFcXkgwb$bp>>3?0E|<4T(P=j| z{3jXm_8*h|g2|pwJnL_r5bft+8PfKXwZk;{Kn2<3@lTJ5kLNyO-Q{UIr(SL?PVKK; zxI>mOux?g?yV_jS2DR;aV7=~`|89oIwFwTi%H9@%P7z8Xm8#l#DDqn<(Nd4N=OubF_l*q?7e06`-!2e+y{ifuiOFxL}a}d%_q53^a z{0T#|G0fM0V)2IRIm?9l_Pak;d;jEF9z*{Z?XkZZ(M=+wwE1Wy`)7i0nnKSiFJ+0b zo_|f!sCR1YzWdwgM4oub0V=?yPy&1XvUM~u8$Tz%L3U#*G56%X_Mccf_JzE;wkO?3 zs_A|=wU;>rzQawJ)Wuy;of)943jZTazZ7XYw^032IQJS1iTaiImqtncY>v1MN7e|2 z*UR>MHUYj+)niod=vHjhC^Nd|E?cRh`OUhSvi5k#4QSXzgRD4De?T*j%oO!@q9>mK#2DnS=Wnh{)n z*5U#sb2koBF8-Br#VvS~2JY0qseC?4=4GxjI<NG zSHG2OW0aBkbnFfw`vPz{ytOSYNpYHo%>w3k40TV12l$#*xm3L?YbQ37!%^GF7GyK# zAvPR)WZi6g9N5%ThOhW58eQj~6*H}&nrj%nLn39J=R-Yf}q4(Wja^o)3|p`YXnd6FGd3+3P)5C?zxAwA~ zHE}a(a2rUk%W%2;%D3_}eSe_{2s}?^hqPmCiWcDdLKyp_MIsxta6e!%;R!jodwBKJo#B~kB z`X-xO8Sf1=bE6OH6FhG~9@l7JScTBVyyAx?|u>m9q6p)-UXdIQRYG z>pu&@h)X|j?$_MD*j{a5{+-WMqu?(_~TZNx6VJQW^ppiQ;5%g z9Jen{Z{cTQN0_#QaUT06v~g2fLE2N>`3X7%Gi(Ot*atm?IwO*T{G!~#|6uM{sl6;D zyX9`z?G!w1qAjg;=DnkLXJ2_q$stYbfwXEU{>AfWn__x@Ib=ESkC}+6`6l%}vj0mN zd-=a0^|_eQ!tuZ8KWRvDUkF3q{(sAdjbG(MLR_})nJmRLulFEj;=~G*BLi$bM;QQl zq5vBXyy5r&d8SD-x*tnztG{w%gkb~7XY8*tMoKvHtXC~ZLOU#3R&sT&-~(hcAvVq2 z7cl7z2v-!Yb8SBa9!zu{fr8!vVXbau7DWU})0+*k#J9h{3)D4c>XkZT#C3RoUNS-G z+Kg(AE7C4@8|qTMg4#h(_|*sz=`0jTm$(D^D|__6pt#d_#KpF0E45-PuM{qbjSeR! z+mE@3D?^Iyn5o{xU$6Q8r-0y{A{*G?=&3Sc>|a}+fKRKH!^$LL&OG1Y?SGQ1x|}rb z_3xlXbT>V64lzi^BT5IwkErl&bT$E+OLE;rViJ2Dul$#GG%h;adMMcq&S1GaeSULkM=lP?3XY7H=CAu6F*pOM#r0DxYGvki8Wy;HJh|FnL_4NPH5@O0!romSjm zC5uvqBU#|4r}$klvMF)3fBs;fdSj8OftwG~`xoRO1@fg&CnbtJ&0?dA%HTlRiV_^| z`&ls-VndNtE0J`ONZ1IDVdvbcJOH^OeMnCi>vhJ&@ejZ^$iFy|Qul@J+IvZk!1|el zY5^|8%(5z5k$%qKh9LqSO3X+el?r5yXp18G!Ms2~EFji<(ha+`QCnb}1vRBO=x+#;m}RArwsy!Ep1y?h-2p*A*gjgrj@r9;BWjOUUe-eVhB;7ScKs)eJCX zF07fmW7K&@(;PesMZ?0ke)R_X&eAF=`BUB`cE}e|6zN3xghNY@hct&%%K?!{w>ZQo z=XipU z(7Axm7!ow++*!T(7g`EFH-E76#ho##{yOrCK(?6C)|T-<4oAIFOg8Yre(M|c|0-Ka zLR6_2Ylv4-HC{(MBwss|l{#CK*v1&P5S96hyk!%UbihZ zu%>Q0^en#V@)Jjuggwv^)34>7y_O1AIw0S5cDfwg zdxii0)&DTOey@8a0N(Qd{EAoCW$WwT-3P9nUa^$-&u=UebA5@O@_E%`-5wK-Y#f1VBzK9s4P21l*`V`7 z_h3IQC^SmsySZ_H#Y%D4oldW>1xoDh?!jbw-HhwKa{^!Xl_V*;L1%_%J(DG*af^BI zjB^Y!S;~o&g3dV6a~8*5Tne|3tKy4gP8cZR~>lx6myeV93U+2Rsa4DiiH120cUYk0rSWuorzcoN;AD+XOE{wu zfK5v1)`YIEZWi$xtHZ&@c$d$d^9&ekfImQ^O70Hz+VYg>BeNbf9^lOC_7IG#@hevl zaash6^!g2a9X`PsG!}xlk@1Q`^VGdRjRG8-HZ^S_%v@qiQ+!TVymsI6uSW1OL~TY| zl;!95cf|{0CY;M!H(}9KS9~+-Kqyw@AD8@gn#xu~_|p*ndH|ySTA@=16M&V}7gY!2 zp4c%a!U=w>9?p}w)Sc+&VeZu$y7w{Ga217^E)?cQu1#9U>Zg+us>*HRx+arD-q0nk zE4pbRjPA;)w|KARLEiu&2IMI0G(j&_Y)y)uJust7P@=|6O}}39g6o0uy*<{2c6+g#V0Lq9_v4~E=zx-ss9WU;i zs9X-N-aej>x=WdP^tFW{uB4t+thP^{Vu_B9?*8*1)NG?ZBgt-X_D}Nr{VF4wD&b=| zQD6$py?oPe)&ni+oqw|34|uh0zCnh6!hfX<@X2Vki*swVQSk)Y7IWo8L1Z0TLWb){ zZKQ_Grd#H~v&5|zlPzt=r`~=B+VHH3Yu@7Hs$Yce;ahu=P!nGXD*uaCNSAUQM>E3` z(;NZ4+*RTg;~4P8^5;v1ymCRO&ND;rmAOY6ovqwcFtED110Bn`34tO*565cDWZZ|% zpnd!A9orcp=bo!n8N7ECpFA+}O%w4?gP-n&i`Se6*3IvQJHe(@3-T>KL^_YI$zs6< zJflB{1*pu6>jSZ0;Q=T7RjB^&w{MO4UB^xNSQ^cAH)Weg~Ot#|I+Gc_iHOg=Qjfv;D()Rhh>VeunZu$XvNfcl+3y4vU~VJQ*7 zpsLE5)HTalwH>fq3m8b>@>gB~$SGr|k1cvYkd8(_3~gGU6oF+bV9Taa0hrtNj%%4Q z3Oqgg0LBO{P3$Ugx6YK{axy_&i5JeV10n6z4Qvf8j-ItF2Tl8L$(IoCN=pE5G>moB zN4MFV76WB-^BqvZFW1M@=@=@v%^?y=@@2+60aESJhpYu|Wy-gOg-1dkMOpav=(Hbj zSKATlo@L`6NSYFgkU_-*>?gXjeUcOWbbUNOEi!mc=V`#0N~86hL`!cyAmvEd${q%sGR13)woO9d4%;5nyJHiKF3}uks3w%!T+u!rFJjW)c~>hFeM7Jp)oVc zSr5o9(&V~=_G?774)F|*L7IfQy;4-0_N#gW?dpWN5Vh0RF7`VOL2F2=REXLX%CjuU z7-8-kvEg=uM+5vgF=ZdrxM&IRLx|PQ?|Xbtz5}aMVI|vVu*&^)p8wa&7uL@b)}=Si zUl#kp3I<#uQJ<3Iiq26kuH}0SmQTqPBKkl0&0-K3Y{B3p{Rqd928Uvek_F>o)rmjK z{@PqFG>|T|wY9Z^*_IPreKdIRl^Ib{AsU~UNTnxzN88L8>MkF6>?>s1V5~kRXOql1 z)h{~J(<#4}qP{;y(DVAC48*L{-CbX->>aqomRdl>qKZlw!wCFhpRSK2=2=f2eL2B= zS@lN=diyzP&~nOA(tS6jDLZ^~hJmpOV3lHdvm zB59eH;*KbbsJQ_G?jR^!K6lQ&=Z|=|@9+CO?*X%Q8zMJxPEki`^SRwW1 zIl1Hpu0W*m>w_K5CS5+hfKn^0SDqhsSS(UP0bj*v$xitEzy);?Es)N<*JSkh4p4Xu zDf~AYG!bEn6kT^@d|#Fg&%D?mUDl1akUM7*fA)`LegfV=oCs{X+{;d)CS0}gMy~b1 zDBo7P8_gUFkyVEF;N%_~iMQQ&=R-SCbgg2(Z8^6~Ob;g01`ZZd{Pt!(yon5`8*|n4 zwu#$)hW5`eX!ri|59P3u2Xy|>VRsOx^d-oS@$t- zdu#H&h@cF&6=C_n%Ny1^0}btW-4U^yT33G33&*ebT2)49Yo5zzA1h6%u#6loR~`AP zTBLfo=ExEEj92Q{^qR1c+69zs9ayc|tI2kmSqh=cScQt+f6NK&@r?H^@r{+=E?Y)z zfbHfVIo^}6cOJ*|-X0j}I zm;_-9QF4|YS~93&$PsUAuVB8hkuSx!6G?jEy$p{MT>r}(k;tct>2JX0^}P*{iAWLM ztMUH&QK5y6_wLMU6xr(37775R|Igg+BVKdL7c#UrX`|kdJdN_d1O{AC0{lvpC%gKA zW0k}F3n7&;%L)GEKNIHff}gomtSCwdMBbJZ_}vyh57VY2OKuut5? zTYAE3QWczQKxL@s6yW%E7+nHOlpFn7p=*n}7L|XXz(~3YLDOq{*B-alWtLq^&h!Dh zVoWL{&lRp(U<>1%;;>eAAYAox$F$x--JOv7>(V(zbf}0d_X8|UL$>Cbu!nw@#GTVi zuNtCzydU)m;FYybk&f?~?tzxb)(*bgRoYm>cP3BB`2(fUDCg=rqQV^57$CG?yUqnW zp7}`0j1fmRFr;^N&rJOWi0N&2jSqT%&`u1MM_ymNy~#_lhl5*lIC`md<0AbSctD_n zmbR&WZ$_ihStljMNJYOdkJ&>a7j|3Izj*K#7DxH4I?v?~tE^ZueZ0)KdHvsJ_QF_7 z)CrBCx9>%psb5GBdrliRUG4~4ea@0T_tcHgS|jloCbG?0OAhF@0y7Bv{+zNhB)1Nq z_dC*Q%XhdboB5-PE-av|lVTdL4vmkgzi1)RXhcI)O3k|w1__WJ5A@X6P9{huktK0!&Fzlw0 z398BfS@VXw%x=(S?aUgB#H~zbeTCZEgiYYdYfOWq^BH<;cWs^F!8w4q9W$D&L`}aq zdwb0bPFJ^SxGP>eLSERd85FCnMSedUkwyN?9jn{^j0V;t6RQ0W{3b!6N95f3kJ>3Cg{9ugzBW+@LY<&3d9&-6 zJoInAE76#VfM0q%oHZ4GPNl)x$gp1#yBiKUX$VcxlgNN*r8U1TfM{W~4;0{r)$?d* zKl>+$aOb|aKI&$}al@8o?gK$gSAw;_RHUJ(tZI>ZG}3zM?jk?>3I8n8c(ZHdFiG2m zRIwE;kLx!!fBi04+Ym>IGuqFhMsqOhAII+@@|r#NLj+Ha3$?Tw<<7Iyv+ctmv;?w! zvZpJ}H3{oitj+Rs60wt@9W>IyzzKFIMgBus-i^Uxw6V2zLIv3aVD1qOEe5?_o%oma z&z!FwRqN7Son9PAvONNue`@c(XB&GN!KC+ytU$`OSI4eVJK{6$!N+IchFaZwul zV(jAkf|xQl%;>rL2hG!o;0U#q;iHLL6C$f#n=8K3ZC%|(W9NPoxrb#A_P)e!eIq-DIQWcivv$Ql{`>(ja*g%ZhoR3R(rer0xhGx&`dNUO8l>l)GmE zV;jwWSt~A6JfN7h$RaRo)WtWUS3#`Cui#B-_vzlbk>JezKPHfm+5YmseW1r6otoqN zARx&%N%@JAus-b%S8h@nM5D2v1Men^W9+6ho%^Lnj`Am}oGP?L+xlaQTk)g&w&k%R zjecop7hSc88mT#SJD|qUAwK%r7q?3ELHY0b>Rm*YWI~fQIF?(f;Odv}wAP6)(K~J) z2gOuXE_14xcIA-_yo~9QL;At;ys?MP5T-k>^JWe`G@YhnW*%d~R5wPB7+6Q#<{bF2 zd3#dXvU2&_@RMZ`In#I6GoMazCybELO5vM%2h_E-UcVR4&a4ON>fod|T%Gxmjm(tL zz;EBGZo6QwliRdJf9pzSiU&MKYhEZ$S*MhA=R^E;5&)fJZ?>2#6}w35OmWy9rBplQ zOJn1}*uSTy$#P3pSPZ6$e7i~|@4AUI>rb8hQPvhC;pG%kTy!|7R|yTmw@M@epP|?Y zb6gs@QhCh0+MYlI;Ew*|=<8{_Sj~+O3r;#X%jfHJ zZDQWaR(MCsqR(WWZv4**FM8xL@wbes1D{w;PCl+rg_fk}n4K6df#8M%HLb#)YKFJ} zmZEE0o}Mv+BV<=s^m7weQts^R_%bO)9+u3Ho|eAU(){Z5D>F?)*meHodDgK8PX4DY zc+54HhG#d zIZ#cc}Y0909fQF&|m2sC{QkDCWW>Av7zi*Kj zuQ^92dDz4jDHd@lB0o5Xv+`s*m%4C{qa4A(mR!HXzFcxw8Ik*n)N61dVp@wJ{S9GE-M0x+iKQ!fsV6;`x8S&Q%=YRylU|0~PG>Y3Q{qp81Q{OIPKoICJ? zj_i9jaoOfR&O(G&X<5Qtt*5nAmWDcqtQ_BMlI$R9f6^J~cePfH9k^TbPwq2Nrn#10 zrPP~MPtSQ?<0h(|%9Y#F0OZvQ!8c=e8lFjhSP>V zz=g0^@+Sv*aR@XUXuuX~)l){Q4bbs;Re{g-Q4jvi>VIP4i;vaTbQ|gWGQ14^<&Uvp z`}>dUrx_9<9Odh@F6Oa1^^5XUSIxq;A6@lA8bkAxQ8>`hs9T}%E~5BvJ9tF2crMyy zJHs|<@y4dCHElK1*}2qRY3FV29AB;JqQ};uUQ83&dj93fFxQ zBeC<^pRoLv=!mD$f}_(@HogoqMlm;7EBo!;H8D;n<)Kx*5D5!4g14GvHqB=1!Q`7s zJ6i5WbKLS^<1L--bV$c+6RsB<{bMYM7olV2-{f*rufQF{b)PMa_4#Yo#miyx5L)tH zyYkc19P;rabpAkfeJqP8*O}>N$OVadf6O}jp}m*wqXy=!hs&ipCty<_pW1;ZkL#Yg zb2Ca=O;uM1HB5bCG=zPP@Fg{|tv=3^a2Y}-KkK-c{oWH<{0d_}yDRwPc^Ly12YDIduwP%QNwiZu;gS-h6Q9>b^I_Quw-vJ2}@Lnd$Op zvW=?E+0iX)l)LYaQWeB65V%gpzYJD|UGPt##&d_O7U#rs?T6zC7$-aV7cbrrR7XMgtp(94njEmt_zGiD^| z4TIQiddw z;zV<9W~Vt_*7$UpP%J_q#L{CU4_hCGMiOd+IgZ&3em=oj@$rrkB9apGJ#Mf?PSp0a zx>w6uSSM!=`Ln^iP`AX|aPzg9!=fwqFRSG3j55!`zr4r#enK$PQV(eM?v)tLwmgm| zk`MyE+k1|&;%qv)RJO-mf6qqNp$6OdTk~y7UB$$!I-P-H} z)?K3j1bI#MgzZ`DL@c{6aqQm~IFd-mZG|v8+`0dfN-wgbqd?yuknUk` zn}W@b^xXB&rn+=$36{7NsOx9~4XXo>G+?a9;*StZISUh?k{^Wu!$Ikofq+9b#|;kr zYI;BLQFp@|WV_Jrd#5=s8;JIJ@8xjwy#az47ju~O5fcUx)V0yR2Gzws9YYC(q(eeM z-MGVp?(%W9WDW@dFUH2*XNh}PqK*g-6oLLA9#5xf5YlMsnSlpLiOl9jNTPTfCwg_7 zBugzEmPbjFGBkVr2$UDKKY)PthJ+8d$evL4%AbWw(cvuLDe@t{+c?MgWS5Cwd2@U% z^QT*S`f)==Pwf&XD|D^G&fm6#v&yKFY*CY-qn2BD)Bo7gJq4Qwj_A0>L~j&`Ftnf- zl+J*I?DnC^;Q`+@Jdm9#hm~l3?Z1(EJD8E)48t^Di?Qgs; z`4x$0)sx*&Q*_vbD3TyJ=|u&^u>;+yk|fk&jVS*`gGL^2v9ji*g~|iuslz+^ m>=!2c_WierEmKz*aev=8)7dI>#r zq!S=uXd%=u?)|#={(k4`+?{igo5_^*Uo&ex&w6IwJ=W8pzsP!#jEsz4^U*^?GBWah zGO}~cG*qM(!n?SeWMoViTvb&cYpSZ=eCz{qa`kW|BMUe;V(x6r*jrbcMMFnHr`ji{ z)pt{ly7T@E$_UBDjF)iTU+8+OH#H3+2N4vUA%M2S?%Q6l*+H&i@fK^EWY*fDt*@s7e&$AYZ^W>r;MKc z9-t_Ho3ib#@*dsOU)0|Y@LU@^452&J*~iqNQ=GkfDJ|L6^JM31BQ#PN$YcZFJm-yE zq=3DQry^73TGYHm{yc};o<6QTf|snv^;tG|c=*?_X9Ce`dZ!5QkW;q(U$Vq4F(t*2 z)0neV+T%ONc1k`0kDhC6%%-1p8$D6E(`*ygoo!OEf3d1MynOf9*K>aNMQNVHM88n< z{8R|=zhWcyE8|>0{iS#G+(!rMzg)fx26D`aUAnEwIrkwkDLi@M%z$X0eHOAQv)p6-i zuRXK5-jZFoNy`|{+E0G?h0>hN>L-o$xeEs`R00y=_sM1H#Z{zb8H8VmKR)lKqU1u3 zP*GT=V)()>O5XNWGJt;htGGFJ`cK}mmp5r>=WgBpanW1##e3O1wEU{9?*R@}7OJuD z6&5MZ-~aHKJmI3F>iNev{LZ^Q_j-`oPqBJI@*MINQ;x~1z%Fh0!^hk#mosQ?y<&WD z<714d+SqNqn<1A!zLkB*dYjqc#b@;?l_@nvmFS!KS9acle=yupnSZbI$R?N4@J3-& zjf(s&9~fwUW;ec3 z#0h4WdYkq@C`a0W+C?#qLMYZ$jSUWiL;D9->HMxVUV>7eTvm#)Q9t|fL{x|6Y4p`L znuWLc??3E)M3Y5@QUE=aP8W74qFSr$S}WJgX-lZ3$m@|>x@Xoct$S%eLgJTd43aqAwDI3d;T3hcm5{6Yy)AVO@k|+Ulk4) zdK4ai)-y_cEM=sW8>G5NAMqVOXZ*}mR(zAOj1m(W9BuUlXE>>Mx9fBp8b8R|FY{H*JxKo*ErWM*Udql!I1pC-0Qg``QW@5 zxX>q0I8?pqzJ@06!vgJ<`^gW@^e~z-+Ju)XcRDT$(uBWa{|5ec?b}7AB%8N3*)#Eg z7}T54=b`W3-r2qees{<2US-n3`}&^ePcJ)(I{mWmaB{I@u^)5Dbxv@uagKf}=)CbX z!})>u1+g0v=S7#qrtQ+69=7&$mh^74W^_*VD0VjYe(U?xI^P=CI`~bzcRMOE!X$en zTPnLYb5DpZQa8FcQYfu74tHykN`az@RpFgd)Ctp(z$MBEmIx^gMGg9PzIFyZVLjEp z%YBA@=lbY{d4z?8mCGKK*_4@h-9}K!q&B@j0uWdkxM9-zWcnA-H_<#I)a6$|cebw2 zh|MSgo&pq5j3_1);@J4O>v-n)_3_DZI{6fNJLOeM*7G(O3@_fJJ-o<7d+CA~?FCvn zn#?P1%$8T2u1GV*Uh`rM<$l3Dz!c$5lri2SbNO2UJ&NXPiT~V|Mr465V3llKf)bV(SueQDDiI@ACCn{tSavvxCyl zxh(l(CG|yjau&;sKS~z&m$(mwtlBl?;|v6&HPG{TQUZ7nSxZtz`mSn+X4= zfs7sH1RAaw5gxaE0+fG+&y{&w*xSUL4XIXJ`Wi&pb{iGzt;6d+*Jbb1FhOR{QbqikuM!nMBUHoMoj%b906Cz=a==0eowz{*oXX&ei z8={++oEOcO5IE3WSnrwWms>e^-{0<6s*rdK=umbMO%agg3Ap`G@ea2Gz*uV6Zq+Hp z)f_0x7<#*z`^C9SlcZd<2@FHQW@KPbrHN&-FFlmr} zh)u#p{F}HF`g3NXe3ZO4Kp}}g*&;3Q{Zgt_hEOIj0hz7ZbpycbTedj$v<$Loo@HmV z6xc(|*^wZo5%yycfBD$t-Cemi!rjBMOjXJ2~`N+A%rjZ9DYEmg_pVJs>_iI(n z63eBBD@PnLCnzRAPt;6fXvO`)k8|?vd(G+(?Hv}@=#$wNbc1v`daw4HS*BQ4S>1Pc zSf9`(q)Q-lKzj&&gw!lkGtfICfOlQkPiBA9sR?ux5@Fx|ut3u}EZGv185~C_1Mm1J z6GZnH_YC*)_c&)@O=obCT+l)|`ej}y;uJ&CPMyVINgt~u2|YOY@D(0@3-|E|Ho^r_ zHh>Pwl_*XDKFRt(jGylI8NDumv@y1C0*r!2#mdBIq`dt;K@5Gjzb4EK!&lnS8u`5i zyiOZVHa(3!eJ#I2D!0XdE{zs{HVZD-N8WGg4ALglxy0I+9cHdJB?Yz;v(6)552&TL zVzhJf_6T!ZgybQ>Pi02v;^|n$?WRGbTZKGoxw)>HvkHayfPeR^aK5@@Wzx0=745}2 zYdv#uQvNt?CuVQ5c58d~D9rpc<>ZO*;qID5>K7xPT z%@xqPT>ED8d%_b{z7qbuLPl@H;kW>AR!2)CdB}r1Cu(-Q-^s5@RZDBLxL=!;kKild zG7}(ND(9RHJXO1Y3D{TBD?f}|;t^ly2mz;DB zTz1`SW!gybfqr9Lk78o;K{I}45nF=UaqkQJ+bR~HMbT`& z75RWPXc!O{n-^=bi&*HI@4V3UF6BpIZqed%Bl&{!hAf7zL#u-qZumL>a^{yXNR6;) z8n^WwMGGyXy3!Su<7RF913xxYEZeynczL^NhPZlIXa#DvW@;}aMwzfB%anVVf<{9M zPeS&LrVncq4+-?Nm}FK&8clDbkY*`PW_Eg_AqJ`1+&wKk`mw)o+%W{5f8?OA1L{1X zh6-L2)0YtYy`f&Tx2>PFclwZ(uB|}J^eO6ODEMfkY_$9vk~r%_{W>BdjJOhEc_sEy z(4xID)db`Bw}DZf@Q#mkIaS}W2HkUNu9U87tGP56cc~r*zq#2ana{d>KM5>N-b6*6 zS{?14aP;0RRpdpB#S`5})?;S5Hhw_s`9~=~I68r@x~W4|s@t;XC0ecXE?oaqlxMj) z@RU9}Ki@RLAKB-wlz}Sxs`*_9uD)<3-bCVun3L~QS~+bK0TXvQwsTgjlU)Ag8%OMJ zFlRX`xp7&6hKeTkEOS2&zhYnXuV0FVYV$WegY^QWf(nl65AS`zb_j$LPNFDYkng=5 zdjU8<%g{*=zPLbZMGuO38*3bM%rU~2$qv6B6$xdB2+xRzr-){nXDrffm82I?>6zu_ z=2PX#<*_++WX~ySL;V$e#oQ&2y!w`WoqfM=>t)DvV+0@9-?ZTK`B45Ap3?B@v6ji3 zdy^t(HkF~SD4#mF`R#*5>>5OBy;cF-;V!vXLQjs#V}s3zx>ULNd*Iz-8dH6XQJ%l{ zr!q_ZKff%CM`&+m-ADN9KLWL#Y@^;#>z>q7tWa}vP5|OKZqs=xgPxVXN z60-5J3vCddB%W-&=^z`$Oj*@5Dd?@UxSi{VBW>#|piN|HsscgyDZt@tnAkJ^dFZ@P zTf{MJNx;m^3ez}BEu9q_bQ0)n0AG9uMytMil`T_(m%vI7z)_Mzv-EI^AAdoy<+tAKkbA!SJ|FyufuAr^-+QMrR{`q=R9n}kas$!bHk&h3 zGg~3TM~(3foxY*T#~c)I&)J8k-aq{M_!sj1+sAYL%uk)aCw@~_XH;8L3yo=H_vAz} z4_$WWig@e(4x*>RTf-yI`!U`!9;h#tktPdFm`^OqP>-+6n29$CXge$M=kZ-GD4`yI zEjP20;8WK-!mcs#$v9nQ?SL_VnD`CA7>mUHU{GQEq!+OOA9=^5roaF6R+iC{Vl=7DeX1X zm@g7>bozMq*Gt|L25L^aeZ`G{Vd>%BgNgW}xxo)F1~Lk6*cYM)+jn@^c(Zjpb$$C- z(CWy~p>q3fgqSZ@=?%@09oogaNd#rUC$o5Isyg8fn&Y`ME{-{v{ue&-#{~=uuZ+md zzY6V{lF=}I;}7B`O9~+KFd_S<%yE-uovi3G5pnjM?4qgF>d)wSu1Dv&qJ zK9_&Sp!3{B*4E+W;w{<^E_n*zYpNVIO7gF;{s;-A$3xnAxq0Rc^o?7p9~?XGFEWo& z(HngD%sR}-`YtI#qDp+1h3cKb*H=&0#G=eEZFqa!@{fC=@>I<`+%7FHaWJMTc3UD* z+~nSfY%DGFt97_j&OSW!xpTY5LV+uztCs5}SEvgwwgx*vWA{ov=;{e`K0i-M@mlfq^h0a4 z>DJ~S6tEv!4qB#KU`=#aNoQzMu*jM9Qx$xfVf8jRk!ob!tyDu{m|7}dGP%|t>;hKX zquYhjRndc3kX(jY)JE}`++z_Fj+O7ppx#*9Y%JsJU#Dp2F@m|84^qg#W z^z?N_sJB$It5JxK?Bhgoaf;dV0&g-HPUwyhRtC`nSH!-1_u$Je?7fTVL=`X@DjLq_ zDLCn5gL|gXHl4k_Eyf+D4u)3^O$`SO1oIhV@1x{lbt6X3aUxyPK*LT{H5tZC_U=bi+%FKqa%q zVQwB_Q$piIUyGyw)AjX(KFz=_29{U-{1`Hz+VAo6ouIcS(fpKSNA~83a7VXD0Rhz z(xfB(Gh-jh=*w!F!(n#^ua--&h?j^uNpy-L?fRa4a3+4S>~reYYX@{1d<%@zj%0hL zzaxov+9v)=r|F=nqBWqwT?k_)3O9$Un_r&C`AhRQSGBwn*o(_hTWG zbi)YASCaHK&9&$?Nvz&FH=P4LGDi3YI!!!voJ&AFyDGO*yhg7MTQyy0=J!@^X5p28 zonN6mex&yYa&$qa^WsdwXvkv7wNCESjB_8uVK3xVKRqb;w!|IDO@L3^1udA*zr4zM z7tiaQIiZQyhNgR^iaj&sujWq|DG*GSE5B!3`_t&DhJm-HmW9h}tGrG}qT8lrT5$vh zB^}}tGO$~pcjV+e`BDKrG3#G8JBiC%TfmxeD4z1|dlR2wRu9J3@yzjXbd$4JbN%kU zv)H?ockT=?^SIl^t45-cQi(!9_)>~^4|@{VdW7e9&c`W_6CRg*iR_f?x|_1| zph!34d|6`JBMxokTbXu3|Dz%6yrulooP|%tufb9GgJf=#FHeSF=?;MLzaW(~@k`R~ zx6{QGXpgAcZ>~6&Ie1-#-6+49&7xdRhPYY&{&U5rx0RpM%}WCCAB5kj@dvqHOQjn> z7g+Nm(Jo%x;I0LDozDWQCZV!kp!bF|A~gA9x>#TCWVA`_zE!ZA|sBCg8EJ?f+`@HMg+ zERwj>x9c$Bgj>od8JF6Fl=#YjnMco0_S6PbIt_zW(7t79l+!4dssapV}on!+)XzQ^Uw?h#bX{y zvO2Qov&4vq;E{-k7WbbH6r=S=_8n5c&;ZQlK|dco-W> z*hHGm%$Ai|Lj5*ziPTD~_uW@AeD&nFns)|=vi58IvX7>Cz3-EeUw%$Td6VoMjS3n0 zIwfj1Pb4o&1SJdf8&JbW3Y0w(KnM2uj)txkgx*s+o@3CbzQq3g+MfY`jlE3cF;T-0 zT{FtoDi7>+A6%2j!bk7+M*t*ovn|Rgm9PGp|9AK3=X+f%S;1aGheO#}rT}86ep#mf zgzNk8|Em`(jkRyDlM5$kvSC5e`v+hHm;|+t@%;zv|nSwgS<+ zU#J119Kjov*tguVkNl&89Ub$;J)()@=uqitzxY;bk=A22 zf~n!8ynxJe@_!xh#Pvdnb=b*XEq%>&*$grm(zsSsJXoN7g7w=S(14+06}LMK>>GVo z`utWhA+e|pgZD?%TcNJ&qouAp%e{$@p*`9nNJ$RxumScn9Hyn=HEFurAmF+Z%c0hc zb3NL}!L61&?QQ(W9pNda#jS#tt2Opbt<4e^NQS@;)vC3UDYA>d_W^iKwn75Br;W)& zs!cUev3 z&nib9t_|q3PjF1d>jMIJ!BgJtF|4arAf(iFNTmiO4_9y%w~2>t1<%&>rp-laiz%NG z(aez7`g(+V4Lo>TX%OE&4PEqJFGa-6Pe1z})9K-3KcW994e6%X@HUX}sgL4_A(u4Y z-RXd#UThnam_U~X-8q^bif{^<0s=6=pwYiOsb;AxD3(v%T?IGctD(gZHd;OOQeV=Q z8iQgR3_e{$Vuzi>%z`@J6?0J1-P)f@nb~W^)nv~4b}wOJHiXfE{Fa6GP$B37vRIj$ z{6^C66So|zVo%`-@oYI>65$Qjo8T<2Xd7d-bbo=XB3|Il5RIDzx>idyHBIBZ64Pr_ zMmMs8)(X{jLSP?o%ev`9S;(VEY9YbpWj~S^nYTUR^-k$*Mqta+#6(?J^d_(sev8Rh zD)ZU#g;owMcT~HYS65NB4+Y;%vheY1=2p7xnL)4`ky56!Jip1rEj_OK?Ty`Dv|@10 z48%c8gBfT$r2Aq!hW!yHba?^W=;@rNkrjB{*m(-YMkR-dbJQLMs`ad$eK+M4QMGK= z@SNFwzDs+y;E3H#RB~O(;&44;R*nNSZ2#EpsfKBpp^lWWaoT{48E|jZ5J>sxYaxs? zXv;HOKlyCE!Szf66L|)c8L!=|;Vn=;g!ASz9QUCMaIN7n5aIZd=Q5$(lSy36)y`oNy`vZl=^I>kerw<>MB5$=Sn`OKh`-U$alK5@7;8eIK` zv_WsbbWIQ0_Z{uOj9J`k4-GF=HM&B@LDhT2Ge<%7Q7iV}W! zw-gS!6fl+$U!R|r#te~8L5--9J`rQfCwCfQ{sdHwjG%!DW z^RVVE^*Ned%_p^3i1MUw$A0~rG(&d}oOY-*qRklerk&ka2VeiAlzPuGy;7lnRK-fC z>7}Qw+VDK)Wyu+*TYB+1X|^chUU?W&dO3E!!L`^|8{gt<0$;LeTK_)e;BR5w?X3K3 z%y^|E9CZ2lmyD}?r!Sfp`fZmXn==N-;j`Iw%v48P-!1GZ5fmgffaR^Vk!n4+%X$k1k~I)$GDW6h$(4?YY6U7AE)jjT8>V4JF)}4 zgIo+2db2|zLF*$0xz5>5Ama$!_N^79EEFTbnXz#$lIJ9bE71M07@h4S8X6D1H?h!t zb^^qrXNYsRgS22Gr`}}>%7mreiXGbwPH(NR8)Zt@K|QrwE8Nbg-*A^=PaeatZDH>L z@hrvHh)5N_}zvq6BWqjUqK<=jf5+GKe1%ZKJWFqoh2z3;HlR8=za9o;7< z)1!$0#d_Jl8cnF z$(q40qw{^UcV1#wP+;32G}ozPdpl(b1?4*;Rjezc- zU}3@s<^uuBjR6~XIA&DZ2e&M@_(C`OSkYE3``F#nZ%Rs70Ji>9Am=p2Y?x6aJ}sVv zYpKL=q{7gWMhL&s&oWlRRzu#{LMxU5@XRgfv^!KjGvGV*VEW7<1}nh;RSVkfEG{%? zSxCEmil<(X84n}Qo{1xgKr`krO=9cr>X1?6ZuS`ua;eRIh{81gS^JE1&v_ENc%n|p zx}{R=I+|(auIITa5*)3nTfN(;io;2oy)5Q2lk@OQCR!@S@F2u{mx zJ5a*;#NRyS`_0GAj_k+2Wno5O0yw!KN(Q7g>$6$dsye7JI6#o+%|u}-uxVCx1ro3+ z%PE@Lvm>zXVV4vlH)5vzpqE*#+6@Ac;Q*ukwyH-5=^@H@GYu8p>P!{Snn7DYgRLNt zmOv}VFY;cQ<^$#-2w)?(#`$d1&;JT;NDUBd+ko%VXF^McsUG%$0$t>*4v4XEuDxDt zlSnY=uqjhb#AelG;eKN)&|*Cjqa$O;v}FXb(#ABOFbtv3(4X?>omDsYc9U~$!}ci# zPdQrQuGHlt1_0m6ra^e0rBtlttnVPh34u$8LAEGBUul6KyMcg8;wD=Peu>#1xogy- z0(zS3FSVA}&ibkzJ|0ko zDp5u97_`cyMAvADCB-_M@$5!XL#;Es*A!qPyCFt9`{Q1nRI|SYoqJcU^yE~^Y31dp zViva?p`=ca8e4?3k1%QH4k$;@1k07Jij_ywVPo=cSI*GcE%M08%`PTo^X5k4?(SmK zxV^IgHe)ALSZQayTr2GCa8@@A@-qu-q5SL9Zf>=nj(cl$V{l0-P@qi-B2^4Qnwe)A zl-bo#9d2Ah-?pB&zU|Pc z+8b*frZe(ZmCHQzJd;B18t|X&fly*V3&p-l_oL01=zw;m@^XxC3C4m__UHo^U=_us zH3g&;a;u=`7YvLIMRqi~vA!;SXLcOa*xPD#aA7iXr5;j#CY+Gj4weQ+joq zU+{#9Uv;qbw35N!lSX_XgBI`jHGQNKPg6re+}?yk=Gl5ZCv%BA)9p?(w6hg?t7heqcygSJOy zumbl;1r%JN-9$E$knp{{)^AV_#{VNZ7jukL!`gE-$)4%n=NU1T0fjkD1Q~I%1eJIrawUYYN zv&L+xkNb|vo08@WL6e?{7>{0!?J8aKAMSIw;iHbuD_@R$7LD}*u$>&oe-A^|3c-O7y)yQ{B| zXhlMM=-tfb(u3(83`*&wcXGqa6Q5#t@T^outlx=VJK=+dpK8m7I%tU;L_znPO1)ehwkTLPg>3jT+u+CjT>%F?%e7TVk*B2-zPUI zP0O0MyXJ)&4+pmb>ui-E?HrKkG~?imZ)wIyH>(sHMprF&O2U>WKUw%%g8B1_DdkH3 zPsK{_Bf8rq8?d;pcdjk|=r!XW^A#Lo7EmtRl$3ziM z`(M{V_d-u54O$m0Tem=UZ2{|brLJdv3X$!skzwLTtM-F#`0rs&QC2<0R*1Yrq{3kA z^m+>`Xeh}OyV~2jr6Qb^ZnmEf43=STG4!>Xkl7XzFJZ6r21-UuM%p$sG=@1h+DA;! z3A4@%Y2@42af-RQ5IkG%olKXXYEBTQxOCBB=AB!evm_)-pFN28J7W&fmp|bw$RE;^ z75^ddpsa+crE9Ze-gSY(mE?weXnI{P9Cq(?%hq6VZFHUsc8_&rWTdy1`(9nk{EESC zlK<;YwpdhW8PTEU&(;kz3WU_?TSa(u5h0t|L!jf9vpAdG_?C=5#re_QM0b!*HgTPH z=$-O0E!KO#n5^Nq_CS~LMGLQQ3pyxycho=q#677LUz?BNOsr@d7Va&5T1#aH2?5K< zwN-KHJqE6=Q2#PT%-V*=MEcDfojLH*5hJUUKtuSdCEP`BsU6 zb!7*{SKpSCp+%5(vMqkZZ`;>4DeLIhp~l=L;sm$*^*t`}HAnyUrL%-kLV3&ktb^f* zTyFmmG2k)2JeQmE=+o}@nVeZqLSF{p#|>A`0wdr^JJF9JgW>q7)Vn4bb79aX!C=1V+j#R+kS|`557t zxOJ~;^-f>))NGA}Aj|LO^aoqfi?AqGH!yRf+aq_~uB2EE@9W_(Dd~TsnBOebhNvs! zS?5SBc8~+_xf+=iBtuMKFbk>j?j`U*Scb$tz6}qPGA@O3jvs^!VmckT!(wJMssQS? z=n}V-eSq(NioFC*enk#-pmWID`OaQiH_&HZ5^qdbUEIyvB%hvd-4L|oR>Uteb9;=# zpL*ibC)oEC%B)Lbtwfrm7%me))U0l{k~-b-BYu#1ak(!q;#Xgyr#ahu3ERNRk|Ol-hxg? zwflh&T`LTMLvNM5de|??N$V01-ztSxW&1mWm4yX&8)xM*y-H86G^>&-s1&xhyiZ-8 zt4Wgw+3FRXNK9I)s}DV!XK;WtrKh8mS4Vn`5Y2}lghKK19z0??=;TSQ!9`0Ee-UhC9kKF2mv~*C<3OG_p zh^@7p)g)@Wy_Oi5y3CUd>jjTs^CXmedqGpOV$PGY%UkOeU<ETL^a5Bc2_aOy z6GTiJC#lw-<&EvFt%Pjy8#ZBjmGhP4{PggT!;7Z7A*6G4Ox{n zhD=1X6{P_5;yt539#6y{--YeVtv9mh1|}f-gxh%v^r{QH;Vgh`2XBw6cRKSE^F)@S z?(P%oL+=Ii5@F$NTPX``T~wc+dN=-rF)838(mn(Rro z7Ikw2jYxUHtwL5#y35%fC^#gL)w_`RowuE*%@Haa-smek6%_I=Oo^``nb%k(zH)*-LNz@S-8ozS~ll*MPKCoy?)?g1&dgyZD3PTSez_ybK34kQ{Ws1 ztn|U;1_9vNwXI=cw)dmnaXsG}h!RfoNgR=R*RzeSorZXg;M_&WYu4Vbj{9^dFUG-oW_L2&z#iMKPpUEN1j+=eB&tGc=C^ z%`T(WL83!l7~?R~2SHtw7>EOZC@I{k$X)aKf>3r!Q>sSQ{ybQ z8jM~Ty>@a~7HmnJN6j~GH2Zkun`|kz#V9V9su!b$GxK_V-TEpk&8^odFP$3AkPC}) zHaqM{wgr{e6{R^^RkaU(GUvmQ`7tgj;=nIP6+H$LTO8LZIRqn?b>PLen@HxWCm!B#Jq`I_L3 zIK4}%;%7FBJs92N238E%v-J< zPcfKuy>PoEedLmaeKEXJ^m27 zi>Rk&bGiHU)pOn)W`psl_DQjbf9_POdHU->uI}Y0c-2?8zt&%-+@=2LCULBm6G`Xu zHX*5z5*}?c^kU4!`^ua$+TXtb_U#P(!yf>hN8Myy{p|Hvm z{!m(tF$wK|k@*}=wC;1NAP+y;e`X-TICuD{^260^w$wI>N&rgveU0^rPf_Jikjt(S2`Y*o06h6Ssj;Ao?YW6N35Z180E^ydq=UF?R zwV@4m^RUvt!N#Q>v}7Ct;A`{`SkiP3VdI zvU;fo8i4J-I1rI0qE(g8VA%R0^Ejkrs)5*Li}!*0;utY2U3-a{73SK{W^7A|`UZUUAwDg6bx*b@<62g;kk;wx+&+<)L|w(W={ za?HYXsVEK>dI^AqY}n>|GYSR(*Afuj zDL;OzIV2kCr#|EO-B0m{8ur!=uFClW2nj0I?ClBhpuFzGRly&6~Aq^rIcpcOm z_Ix6uboBJ4nK&)!_|^d4IFO=#0oOJEaMI=49utV2%~oTincr-B2X|?uZ)yQESn~VDK%lPj1RKY)4`*y`d9g7s5+vXzIlDDV-Z?X#&6oCQi>a?Y|qt?O2 z&0AvXS(pG*=u)KOyu7Jx9*L&c$^PzFYX&0GVh1xPZj033M#u?-*s0~-D%Ifdn*rP8 z2Syi;{SO>S@WL5B3d!MhWoa(&J+X8_PC0lBHCaSMr8S|;_Y@+mQ8SqA#PZp0AusEd z<*v%8%7sS3uvA?=VUIs?VXjAB%I5Ojfj$C=(qz4GAZ<>bfqept{r2Y^ucm%6r}slk z_36q>r9v5poy!3Gs+a!wuU4WdqHB-o}QjPwg7->LThdL zeffAIOI_@57BD^v+>VkKi$z4qZwWMsvtgT&z3@^*RO6~uZLC<$KK{;*So6%{OdE;Q zH1S2bbJ_l9?6UQ?%w!P@SlVV~hiEP*J(Dc=$K>z4qHZ}6E%g3m;n-`Bt2fwIvCelX z{`|Y6{^T<3Sr41!ngxJd65S11-eCDVEFY`o>?!9OUZ;_xWP10U)v1MdQ!$ve=YOv2 z)_$%bn?%?c|7RUinqZ*}cx6xLQvXjFkTy|-6#q*~!Ty*26WCOwHnE&J@_(Bunn|8c z{(ns8lGFbD4ecLWNh@q$%?Y=esdirZlS@cT*qOrt zwf+CzxbPA7Y+91!kuy#tNgmGtxOYo9Zrj--BNSj?8YgiZ@Q<_1rO7QIInD#Cg`FD< zDo&LxfPJ-pk2YEO<^_l8@J#^DM`NStDaKa3F%mIV@NM%?XgpWp3?CVL)e5FF`Zs)` z6st=E?W6vk>Y?69uaRj;E8$9_l{m;=zBGMf5dEuB&&SKoe?1jgPAtuss z-5TyYi8|iRdg6bnV1MnCC?&ZC?AbxgXZ$<*4~W9FVB7Sg+K}eIVjQK8GXf`!uzMcM zkMf`KA!_}W%R_j~6w{BoCj$ce6nT@k@4*P2opYXzZ|4^N%ZzXLH-^_e;C0P)sx{c| zOL{J~aG2(K*Wq989Djv{lq6>8MS7O(;S5)oktJ9W=pWoj5lle&E zTfQ0z@i{)k&`ZL2{3w$|{|S#LeKaIA9}j7TPgx?Z^U^GYlcz*Y1=zWO_aQzS?L z?CXCs;rGSH|KMA6meO>gy6GNdJZo-`J;`%R-uK^4B@1^VPY?^DE;&96<1V_{bEnt*vq+*0ZK^Y@n=DJ!s2}cW2Lx z*wIqL86Sv*{UMWovycy8o}MKj=t3m!rK(G1%^w!1KW8x|KmQN%N=exlR3Y1-GJ9h)$6+Qb^A~r>!)`8 zX7r2yc@38$6yC?WI%b_D)gdSS?PHrIeDNQ+`^)2Z!xKN<2kxBzjNcPaKT;p{sM+6- z3$!hVc{`Xn8YS}b>XqDGe(VJ@OV;22@5pZ5C%aa%No?nu{v6!j`=tU1@8g{7(0_OL zUsK5_-~3LF_0Aeg|8bARO%ijXSk3*<`#u(7QkI^};z;~Y4()kPiV?b;_^bcynhJ@4 zn4Xk-S9=?0(;Qwb{Q^gd!fMVyls5d-IbNr1AQwZ=_`xk%A{{r z^*50owCcIXHq<-m2$mU1w~wzD5p(NGzV*Kh-EpvBm4l6 z)SEbqzHSScY1?S5XRPK=MpQRjwzC#3R>g;=1sj$LYm5MyCED}*Y#Ynw`8Y|p!ersX zG{0yxXd@3Fd(IiUxpC3NhmurH%+-TH*KeG!iyQ|}bgLJI(F1G4SyG2ccVXZVs>cQ} zQ&*e@?J2VEDo>Nr+O+SPw63)ThGtn65G9H4UT#Qgm05cWHK=2k$_;$Rg?4D790Ab% zOW{PGiZm;m5bmLLBmpL!zvFLEUW>rvc`^l1uxq}7Y(uWo4lDLST)nzn_5sWkm(QNu6}{* z?LmF&@iSP)Fy#Zrg)PP zYvy9j5 z=`BFHCV&oS{T!@yQ%Qwq8Nf!l$aSJ2{1;ib2IO|P6}F!X?- z)N3sh5Du_qgD$7Nvd#uu3ODU?N^4FZ{3XA9nZL^CnX`9m&}ouXQ*){4R(`Tg&lvO9 z&WWn-4_rSkzSr}+R=D26(6Ikm$vvUDo7KmBacpro_pp{LCeA9l%x7s>!acp-%==Ea-MB7sI65AXN!Y0* zNjD;Yp(yl3-;SF_FMsC*16pl|1P+57& zLmWV3K|{g_m5YDjzz|Ok;i;F6PP!!i*6-((f+rj7F)b|bRyMvhFomISA=~QYy6sw7NIG`QXE@74 zY3P<2_@KEmpPg;XR@#h?OQEYqIt*mSRIZ)6l<5y>kr~& zkA^$#=(e2YiCw~LcgxxA>8gdINJ_|V*R7^|Y@{nUzP?;I8gA|=71$(>3`uWmluZwL zq71s&NONehf#GVaVYwH9)3;61-j)trT3D#T69TV)R~I4Rn`4%3W+h>JNlRUou3awy zu;-ObAmwC)pZvN-cJg*H3ZO2Q;v~HEd7L$E4W$S44LbR%@UaJ}*N4(Ex ztgI)MzeFdJqhM%4u38ZI_k)6;I<;Pm5S~qf{0h3;8r)g3>vKIFt=j-2i;A+D7u;;T zo#{`9{E*ySJBX6p-_*kJ4Gv&#&K&GL-^HgtL_*PW(Hty3s^FI?lKU`_n4 zu(Dhmx-eZGsL13XKMLOp6Y2*0rPY1+~Q-B@ua@a!0V1f~nGW5uS1pub!gp zk#M-O{ai*_ckfAjsvS46k4&s6a?y4$&WmMTgu%4q#;Y?9YEVMmqb=+`_YAYG!^-45 z4D>bscWRGa#oE{7({*T&bu|@j*{lHOn9t5LFg63*iBS*lv2Zd3Bdnj)^3k@(%Qb6u zvg{XZbz~kGHq#5KXm2}HSd4kcLT?W6>2^k#T=zckjI86RFy5^+?52S3ak|Fkp%bE7 zthc3K`&rLP|I+_Z-i=&c;> zqMQwZ1uj>0?vnPvftGID0LR%SL&2yOt4Lu_GB$s?_I$xE`7GNRD ztIsaeXNm|ts@HjdG*<~;-o!^c#e4Djk6`Y2h+j+$M$ES0!MtIZyO7d8pR^s5Ed=bn zSF-2{Zh&>+HM`zg?cP^zmq=@oK!e+GrRTwJti!_bvu5V2n7)j#;GJL(hK!J0sP^#88Y-9{LFU|@QJBs81phF1C)PLL_clfo zS+c`|95ZlnaW~XnA8E0&S+!y4hwI{p2ji$w3Sf=u9H1CiqldA6D1L(6MMV%_bS${N zJ2E9Mq*7oQBH4oz^jch0stMxnG4O^(hd(E+7cvK5A`-1J^Wq&{m|2o&HK%5knA8j^ zkwge5uhMg>xVTatiJRB)FUD=3r|UprWD&_!Q3>{=o;i+Xp-ykx4!TMaV;!cGvvF7~ zgr%ncoZgznzYq6ZcJ+F40PeiZd%LWn+Q6k+&6O zH501!(rH-{M+bT#IUXuwbhLvaTMlt}(XEnV!rVe_9%aoza#@5?h`j?eC>vnNdz7zE ztBmrrfP{QQ`18x$gHW^Un+h8(b*%;+OdfXJ!)O3es&eS$CRmTN$R(F^$oko6o@aVd z8OL5XQ;Zqecx3JrWy6K6#Dj}@b-UAb%=Kf?04g-(+AKbiaW@PI+O|DswpaT@k?2-- zQ2R9iK%yY+4ff89pRLV9j+l*#lV8f(UDnpy2=J2P-Uruiwu9zYKO`D(dNh6Z&uVL!t7L-`L7E)Dk zmaO|ZYCe!vlSe1#jIq0ZyJIUg`oLKH1Rkg}eBZoXx~OIV4Q_uA^`uP=#b|b+r@rX) zWuV1*5&7Qr4vx*#<>lzs4UwU;fO@YQ>HZB}q;N6px}>@0B_5t&s$~Ep_m~pLt`TB; zfG}Xs57e|DYv2RmU)lopK_yRxwFG-LA4C90YryqhQ2B1a-gCt} zK#cnF+IMdwHPsmJ2S!Y1eBM51BG1z4#eS1*h^Tw0z1|#dA*3IoDG__CJ2& zKqBptQ23j-_?b&LzHNLaDU)SU+2S3eel&%fex%KcE>c^nR&y|&c6yAJ9~K(+0Ac^f zy^WgaEd?T`jo&p0e6W^b-sDA_LMEFH_PAn_-L4|3Pk&eOyU|$*D|HUv$vlFG4Cg*u zvZ;iEQf^tngu>!ys#EEk18E4uk0;YsWNBDKSW&^WNScJ+?jb{Anw>8p$ zc+rcibFhIp)G_;}wzO`B4U>AJVY5wCO@6*L-P11E&RdDUX{wGhQLgo3X&-mI7$7BS z4oe*t7?uGw#Gp1j=>Nk;fDl^gRpTeI-T_i@SM0Q4&j!*Q_T>*W4rIVO21JPhz10`T zi>W%nE(npw`Ih3@zer=Pq>%4swWhGC+DQD1lV~4 zuRd`8y%hMFJB2yW)C_2@xtUzt5Red-Ct=Mz_B%2qv}U6w@cL#^=!Zo)Q;#NP3&!FL zPr*EGOz&Jss&o3#io8kD_KdB;?^XxjD|$Ngib^)!MzcT{YT8`bIQUQfQ~{}}K(8S) zvQ^kxC)YX(LY6Pro_o+SqYUon2#{+>8~oT-YRMZJ8c(% z9REx?5S-tZI$Oua+73PiQ+eLL_x9XK5=qh^|EKEfc?UGi;hFap@_#z;@G6~C5&M>^ zQTn4Q0NBqAR|YB@Hxc3CK#;L)^RC+z?4N()6)nnL5Ck4q${Y$693N@ft^`>9KV<$w|Qsk}e@yRv|Dhz5qDv3~9U z=;ReHyu8T$2tSMBi^E{LW@$}^_Abc^ElUI@&RsRrQ>~$dzUr#!$RwD+hyD)Nh>cjZ z;jH)3-W<8vYX0Idac5d`_z3cxawFAi_Yr+QCC&>zbp^_ z-$VyX|2$nJ9lRp-qtbXvBE(0u^UYAyMGG6eI9oWpd8^(hiMF+M%V|%M5@U#c8aCEv zy}svk*x9{Lknl)?aD(>B+L`rNadZ9re?=mGx39F-pM@dUyEL_t$g~JPXs&{ZuJS-u z6=zSL=}3)mXU^q~|GsT@yEn%#DQ!5BqWAWYIEU8Eo8L4i;72T3o)`KJCF+HC73fH% zNSrx(IFNLc;m?$IQu&efNL5(b<@K$69A05pxjaK#nuE+y83a*BE z+zk>vy_WY-CvJb+H0!k6=RR@!vqevCoHCi(6C=k&eE7?ugchYJX+| z_-OO-_wSNv{qlZmvKRz&Pu}J%tm{lK=5IC*Kmxg z`a!|8z-(@RPfl0@Mpd(`?WaK-6`kMmBpDH6kKZS6ZI-L6uPMNfY9`#~#n`Z9Vo#)F zB30Pt46@{e_?j?8R7s~fr3>eXO=<`S-_H?JCrhEooH^A&A#fd8zVZCfB4IbGGV#bn zCJWu}mNeMjOkrnHb^&Qu@%8W9B}=G(@Bq0HqIhfn>gBq=oSVUbT7+#=-4>+|X^*!} zLqHAm>Dq1WT=mecGp-8t^?JuEu(TcC&YVs;-r8_e!N=Tj;c6KO@6zCMWkN9=T#hx@ zrc^xg4$EoBFvTtQdK%Dfs$U9TnEp0`vxRAw>LSYbA(TRLmchl zvuarRTI&HtE|7$uB-br56Ek20LLQ_)%1pc0yj`t1;kM^#(;S@#m|d4Pj#_^Geus(; zz=I4X(PRE-0e}^1W9?mpGKSb!D%r>R;Q+jyF%AA>J6abgCJ1xJ@RKB#TF35wXH*+vVG;jt;TA+A> zaofKL+>|7#Appo4Zg~=^U8f+wZ&kpW{Rs}^p+PTxg`k}cI|v;EW?dz31=>B_G*J*9 z&t;u41zi6grn$W+d$U56q6sTq(@{wdBiGy(Ow^A~Zf}+~iO%o#i8RnGItjb4=ui^r z0)VUKb_<7aq)lC$3CN2AI@|9VQFYnd-c`$0MU*3PTzJ1DFap3(ydp)NrcC&giK{&i zJLguiE&#q>%9ps<_ib;(`eKuI@lgl!fcW)7OcP%{%TkYYcXFU8cq&8KY%ak>9UE|o zNbUuO9Ch|&(~VzkvrF-gGFq7SOo*(8OIPRw38`?w;ME-h_6{TWs3T#Q=jf1?bMvWK z%nS@jw;DQyS~EI%UhoKJ>x2^$3%1c7x3M*C{8{{TrH!$7 zF@Us$afo_Bc?|i!?!(--va%xa4Ylsd^6EuX7Z>@4N_>5PoKo2%kc*zYSP7yVt4!dd zUA6bDF{T@S#DKv;4&mB%&`i^u9&N_NiXBfR20$zijFaeS_q0MOnOZ{2s{}XH#RoIX z7MP>*Lm@mKrNM$~P>R->;!xgNl_RtA$hgn~01Ax-REkt)4*A~%A6qzqQk7UoE$3i3 zXY)*sk-Ozvf}XSsI8*cKZ7}7ph}lv#1#_<2+rwST$d$N*CTdud-#6|DzpuqZ*`L0n zm)vhjUYKw<;J47@*9jDBe-fv9gEfShh^lL>7Y^Zavz>-|Jzsj}3Z7i}ozW=y{S*FN z=?wer({TXeAGZ{$9a~@yYF>@qHdvE3_xj_bwt(=WT6^rysVJ0M)bEY=>!wMU)=W4#$s_iiv;ytq| za=(J+kb&7gvTRr%QEs;Fax+=y&#y`u#S?2~-h?N7GwX_OUqTDLp8moRsd|$ncanc? zGovmp-&k0trYd-;Ahi8PEAEaB4TW^Q1N~v{mKpBCRyRU7Qbp~w# zW%`r0db6T69^;M% z!ABa(KF!ezA;O=zgwx7hi0<^kxQD*p`{ow?!cWlJi z{rC2&!kIkiTXjqf_T62aRx{z|D4GKU&sBA*8YK*izb$&|)*lCBBn;F+%@?0DNidgE z4w)kRH)=2+RUBUiTJEuBOs1m+jka!VlsnCa`UC-LHN2+H>gt>(USy+SYGx+>s~qa$ zdcu8%qd$(MQ_?T*+(`L{3^}r<1N(gBg+c1!qvjOOVrLh)qiu(=+ijbihG06w*4UkR zCn&9<(|=Gy3G=&gDgC(m4{DqDMmOd-ye=oxA@@co2AyS7R(P}xXK(#QS_W9(4NA3= z``8dIe+-03rkuoExW3B{Z%}VL&9_iaiEm6xvft+Y9n@JsJMdP3Lx${SP*CW@uSmnh z(1n@l@ib7Im~11blU{F78WknsNhAr2B5d^BSLIiI&A05W zTVE-ZzdvR%=B8_CCfJ4zXe@>1dK!*|SsacefEG1=3DJV3j9A=K4oPfo$_%uk_-aIl zIl+ZUyr#B&fr?0JL!}nodP!NbqGYI=fHIRHZw+$_M!7;wNN#$&FoJ~R=S=9(5%F84 z;MNvijf3IaX81_V9(QCx&ce3R{R#&)l!w@kM{a-gSE!GTYs{e`viW?!;w2Er zxQsl}M^a0l#FM!#^Hi8V)Z-I@rNd9eB)(3Z@NI;9xQ(X(lV818@C}T@b|fR)TO(C< zL-@po!^L8n-zy1xDVvQ{@-WLS+hW0tIrpjHY3g8!ovjh7>fFNsv^x-zD&B!}ic@hM zmi3McZSPpR(X{_{xOr?M`L~b{P}AG?lf;2br$&OdL3yIh+$ zF-ARs!QgNkj={8;s;b)YJG`CR+Jgp!YBetnt{~73b6YdKVE62qqXn(qZq$PMC!1J9T?k@{I(X|pO_+>rCl$HA4f zx8o7M(e-!jXuYXKIQ<936~_Ct0nWk7CYR3eo+Xmj>(_P%cB9jd^lMF8opN$58@8_+ z8ob$&UJW@O(W{)ZIWCZH&xyr$3EF8^{}Jhe)zYQKMYYi4I4!a-zq46qStBeNPz$hl z_P*WY(8ep0IygMV&hYlgW=?!K(n)W>-=-0xua&x0e;HVy*67(Ge!Bw}$%Y0N{e{%< zN=u&NV3Vz2!kCMCLb0a?(z&YY51iK-3Dn-6DaT&SBHS7vF-(Mog&x$wIu~4uQ6;>I z@Y+Pvpl7L=(UITsv~uC>(%9I@a921m(h71)Q;mGl5<2RacDw?4^;A)2M|)| zIe`NHY^fvw7;T0DIc}AX^{d7!=f7k)y!A>k|KmI^Uya>kq6>KZ?A%|_;{-Nk1rpn1 z3z;`(SyF0x%Kw!Ft;Mj>|2oMEw@#F%3s9ne{&O@}ETsX#8_e?A#sBUCd=HS79x?y( z&!67{ZY)S_J|v~O@9`uUg%XrvDNDrHv*eQ>iV+K$GrfiY zC;kkIrvy>k?H1t-qfPUe=C|MG?6-MU_cx}{2Y_zpJl(;u$3bpcwJOXNWr4h~S-(YSD? z`#0GsVF`SP5i?cl~!F5__)C6|Wyfb$efKj+bL-{&N;z3T$NY zW?MYVUjXBWSP&dwwW#J$o2WhX273h{kV6b2X`A*5p5BHKJBMjeAw&f_^n++;{c<|p zh9+lo;6`OkB8Wn|JY$nDN|Ci{*!?%@O4-EC6P4IL3*bUtpn*~5al+!7%X>ugJ=$5# zT6VGKU~nY5XGqg&SVc5UeKSpD{dWKk3@3+Y$oHPt9%CoWeu(-T=Ht?Ukmz(m#DNps zDfTWT45re}PCJ|3FVv2PT$~;fSK)H0x`FD>u=0&steN4r;@bUTP!jsdoZ;Hd#fj|S^$EQ)Mk|0L z%K7dYw(={m`4elpER~9;A5O6F+)&67vwNNp@X%!cx;3!#_P^+a={2V z5$sd)Z?2K~O}+*<4r;iM*Y@D>0t(B*Zf2aL<{Dqd8zI9=IzfQ0!v{N&-qs4YuV%T|q4O{R3vta`n5YMI)NXU)b zcX(f%CD=1cmNx27@4=Xe#>YhqWMo@>4~=>@A4nS9| zVn<%a?@vHX4$MotxlNrsWfiu^{`U!TFNQnOR}euX;_sKIDqdIbakJ ze|7WlbmD`6$Jq}8K7*ZHbnyLe*loBo$UgL7Rn{SFI{ z#aL&}0LS^fQq}fod7EK{0^UWZsaVv(jex5}XgMQkpu=5K)qbrl&lx=6%s9{bW%y_2 zeA=+RA*~z(&?~leAs%kTEh2#G5(jsgpkW3Har<1M=JR_M{(OF&Y*fkiBS1X-t~lJ% z+t&f`0&bL$ZdfRs%Ia%@A@d&k;#eId>KH7!^(eGrqY=zI$J;w5+{ao z#eYgC=gH+QP{vAz)Iv54Xjg5(4zS_`^ubc`0a$L-6JEdvMM3dZ785aVcJf7ecq(=s z#sw`fcu#Lj9^U#b%?f{p`}EA`%y_AB5e+Y{u#AnD}HfV~5?X zcdFmNGtupFZ<<(zj`In*Z8}{;8wPr(%uCCHEYZrZ#{n05qJg8S+ffn&ooL`ff7ZVw zs`~Em7g&81Vz6H6aJmj{cQoM)O7x~)IGb&)+Q(9`{a{EitcS|Pb6j|{U^-O&EO?KY zMFOR8h-42%IiM{NGs;KktmG}Fk==@-e9_=y{=vP+KtXYiPTOrjm$#I;DtGJ|M8gIf z5_;M{+rjBSJhXQ=-1bn<2ZD7%inq4x^Yj~dQAJhSp))(7wLlm%7}w$YHWf^ohu?kn z-DPA{Sa4%|9Z(aiXY3pmhr22Z_Iow(#ZB>yWYvw-hxiSdi62bVP0VbRc&O9ubpRy_ zgUpHfso=%ZFjp)xm0!%aS=fblhLu)d-go%h{Iac~g$==hJGp7fX4y#rJMC51UZ?!) z%9BbQS?{k_^vj(UkAai#3mQ;cY7_R>z_q*IX*vScSbsqy?0yi?k3xT4<;{;oX3$e^ELb^ZN36HV54*By+|c;VP>9%h zN4Kx?BGnJHU{; zpYRg~nZzY-0dZ&q z*H5kV2LK^)c6dq%ZhCSVb`_TpkTL9(*c?ztLk4Xv#VS{&QIPP83x9|D$%UW>wR{ILTFw zTz7vr9@Niw0(Afg70^?o4Yma@nV<9j3{2b*APcuW7+G50;2ZX+N>;Q_MTAq{oSfkq zsKoQ8Z*I4%rrI9pZ5GH0ydN?GSoY{9p=3?LVi?UVG$ylMn^3-OX^-WBHpe#$E2#hQ zhOJ;mf`9Yy4w2Z_)xoK+&5;nnjl|}pXmx=s7NEb(s*O`aOvBGFVYaglvoai}Fp1h! z>#f=6e=^qI*MM1`yHj?qk2|!lHD1wM)^N0JD(l(yQOBxFk;gWEAZdRrM?pYEY^QWu z|7V`Awsc*8m`2TJgm7PsHlSVu?WwsXB_39>Z_ zn@^*-ptD2qKV|9f&eOqPoAfJxX;H&DObZI6WK_^~446vV(Zdd~*)f7H+QW-0C>p)| z+$G!$Ws}fUknY?oJ3KK9tkk^WEc=x+0)+$M0qO=+?Fj47oeL%{tv)+0{{ZHwGmzJ! z+hs$0%ngf%Zad%fQZv>C-tvevhpV|4l9O)9zF;Zm=|8CeiFwm_i^kGMWiVyr)0eV% zE&FNmdm~Pb-J@v6T;=CVf1N@hx z6b@WS)b<1(gH

Xa6dqOJKRNrx=6eJu^4`LUtXOwgZ|IKeVT>EAlzVULtk1gVe@ zqUJhGb7O(_B24E`mljNFEm?i-S79An#oMlt%Az^zA)#4Alu$u1 z0^?O>i{9#wt6L~5D9In#tDR7Bh;Ef@uT9)ZHnj_V!EJM?46pprw78@X2U~{4Pqk@- z_4Mfs`+;-&r{KJ) zxzX%mo8om(kmN~w8}QGB?2=#Or*oc^%{lA9<0(Y)4=)vAt%`1O{{@hAna+@ET%F&( z8kWQO?sqEjGrjKtS3I;J8o`&lJt zLox@t`RPT2DWXes@-|I{PR5iWdy*xpr+%0Ga^L=<5oLpYsTCBq^Wt#n{FXyT6-x8i z;!}@>s+9wKn?!K)Ft6B8{@Txhv@^}GfJ1{U*!$fAdWjjeX?VwQ^qkeiZl}IP@YYD| zAVhdTH-eSt=U?lj_0#lb(<{lTqPV&G#XV85?^kfAc96qMXm;7kJqgbA=}siP_2#FG zc-QMWVs>KkxRY&XSoQPMg{rU_!LG%ac^J+V90refY*%rBC79FAszH*`&XH*DMYOGw zmU$-iguE+QH38kS?Om6E|eoEvOGC@ zFp{?U|71q$DnXE7E4RLv2-F`7PN0B;Tsidsy>Wl<@nku}a@WCZu8y)v9)AJyAbrON zpl^nM_1BO(Cf4$PTqoi+{1M1p@wewCZv@l}d1r-t*=_Xvco!wT?r7`es5Q){8eOjd z!Rt|1U>&Tp(dq6kg39~2r;#yO3II^Nx!=G=Fr-ED2VY%R)Ls zS}$7q#KR30Vb{kjx?UgrG^*aP#!gzx{gj~q7fwTvv-H6g)Ts8D1+q*>fHz_Sb6Xb) zS1zvtTJPM{2A&2+%S3n^jDJeOO3)k@Xh%-HA7qMaE{GSJ4JG7=e)%*PSZ`Pm-HKrv z7WRPCt%7DdJ{$u6@0axgd6RoPV?c{IvOX-ax&#7 zK}$1#^fc-`&^xY|+IHsC3Fm#nkpV)!Cj{x&o4V|pKbbtKJ1GtBa#hfMu@Ol3FKo1h z>E5PAN-GB~N|xz6@*hcPsrP?kFn?8`#M0l_|1?RJA`hghD^~@xPw`Q}_s)6lTC|-m z&-4VuB;!oBPybMeCjnJ5prKmpc}M>Tu>#)PdIT_8^54=DupiM2{VE@;DxZf-BndP(&at#p4 zkRJn&y!h+IUr2=mmwLTcX8(KtfBb%22P(vmZWjL(2>hO92H?^iQl!SOmp@0-bb=rp z@SBpHdQSq7-2C5(`@a+Sd%HIN-yQe=V^h`fp5=fJn~0wFUs6Wjh_&phW@E~p+50V? z%6V_~g8T;0l3d_5kdgeMJJv#b*xA1n- z!ufC?OV5sJP~wVOt)L*PfC0<;DZgUk{lqRSN(EX?cpOO6%B8Q>?{+i?daUZ+FC^e- zb*Vh%90vyKD?z{Yl7V~l=Wzh=Hfwu5&^VuYd0$cb{ld)j0nl-t^YQ*9tHi;X%uKHv zhZtVjWVq@2_5zO~Vq$-x%y|%cU`!xjzzE1DNdtmm_r*TOuKS35Qy#n#aq@VUjFKL( zSfbwjgdw((GLzOY=Pm(E8V9PDFQ2jtV*=X$=KNJney}C?Da7qidBJY9JVOyo04;=1 zZfP$_uX=b9THOxSJ=;|etUbq95A{43uY=TQs~;S_`PlAzh?{(WG}UE4aw|M|)ni?; zzodS8bU?^^Ks`0bx$fjKH3P~*>5AM{X&UyQS;6ap3$L%MP#3ZB6EMEVPYAhg^+JaD zb#g?uP#eQWR%_;}2cEy*gki6+pM}7myr-nLFO*JeSNZG})Iet%-nA{1kxVGl&YTVt z;j5e1PmWQk!dNYI)7SK`O!1tB?sb$c93~TN3G1MP73U#u6-BpE&kAbeQlBQ=kvxHC zPLe;-1AiTOkcnGbscXt^u*~|_r}=Y>dU}fZJ1@JR@fI5#19lqK;|l8Hyjhz4+? z(=kc-)EbQG4A9W|Zqi@06F<4%BpFwKNOdHfc+}gPu4+)vU$-YAHV{K@uWB)I zoV9fruCzd_s;f9B`ThMAM>@Iql=6b=@zKYvy2+jXEjQnNtn=i4f|_2gK+iD&bu4Nf zpG^3g$)Zms)cJJXNtJ0oU%w*5saqt|a%eRfm_5}H+vJV+nLrm}aP@si^x7f{&Q*s; zO(Ze6tsM4G8tjItZPwOVemh33?v`&6B)ou*`&e8*t^l%{zO4i;R-`Q$SPu};-ObZa7@v|C@ z812WGjrRL`ZH8y^h_GgFt7!c|zCX&L~j*bto`TG6*pvt>- z^I}2An!YlRcOZrem?a%W=wx(%%yifz)m_!O^jy?##TF~d%aD*iN$7g&K6Fp7WcRJS zxbauE5&F4k#&Q7|zVfX^!E@jNPT@2y@)C<`OnW4fyPx_J`!eKjzVD5nv#6UlTkx&l z^ImA?IgHuz*FJP47&}ny%|xj(g!Xu?lnmrfJbyr#XXQ~P_i5C1KBX0uqCk3 zA7of3$Zli=QlQ7iO0jxlad{z#$3SyzeLO}*VlSlBd%t^Y7Uh#~3DNH7m8XxY_t}S~ ziVaRe$MTN`BzF_HQi6`vQx03=^E~ROf}tu6n*_HE&~bgTDrmQnBv>BStz)0$HUV^w z*6(@La6C12HIfUegfQ4`7%&0thAuckcOQ!$eKQ%jyTj>AXR%s<94Jp!RR9qX7cYi+ z1=rcBVho{UXQ0CgT7t0076}fM&f&)mbsT4MbM!`eHlIbRo%z?4@0{!nrvi-|&l_dX z2>}J_vjQ9ejC=u%`}}poDH#3iY_e`MPp6#H{=6HX&Xb;rejpPk()djHV6uXI-(A?Q zw*y`+f%8ggn($gKYp5HlsR&;s;A`q~x|wNPENiA#r;GtDk%uf@HW;j68?XYa;RF7{3dz@K_A=-2jI5V`grebFv|wq8oo zIQh7M*{*ph*1{nHKUeGa&qX=^$FDd=Ygt)Hw

@$7F8Zx^jh-l$e;Pg7IBK{F}Bn zea}m5rCN<_V&h{!A~J6>KJ>G+@G4!u_yrrqx~|FiPP0JvE&mm)$(c-u$#dI5QzpmM z%^sO2`C6}-I_SL3>h2#4Pv!JkK2cX^QV@xK!mn%YNKZ-WKQ?Ayz)a6)lWK2)P>7=C zeFb^_VC;$9dJz@}XTRFZp_=DS%bPs9)4v!xWE6Yti&$XC70J~ z$l&fqf^Qv!5iL#PKV|w+GgBuk{bcN(reM!3t7uX%mMZj@$<$ zWe<=;TQAvSUu!3x+aB@Md5X-jzwxXogPsr{4AJ|?OI}{5_j`W544LQEbf;>z8g64$ zZl3JXk3``mQTiSsxs7cVyAr4MfMpFPhW62<*B3GQ9AEsDk>6=VXbi{Wy&yDnI*(^O zrr4Kg48qi|gQQVs5La-iEteU4nl~}ev|F7+JmGggo`Xqin-A@@y?#B-aE8l{>G#qS zoiQO^w8pVnXUKAhT^Ao8zqB&MvQ}y@MH_wo>046hZGq3et~P6iIp+k^3Lf3(e?UU+ zrJ<>%cB5QQ?cBTQXy$f);r>p7MOe;8c>5EMpr?Fs?A+Y`b#-+&-k39X5sTfscST7` z#Bn^5*MmW1)=I*o{-IyzOE_2k!Inv&OoOp~aBI?A`MQh?N6+ZjpP;ydmdEP7sT>so zV7BK`b$<1sZ=%^%4c21AY{&}kj1S5t&l1K!CewUF&f1!q3Oh3|ELhj#$Iu8#45hgY{Gyt=#RIVOYbU$0Yqi@W#_nF_Q9Sm= zn$-w7^;d6D^}!+~5b4az%O8q~KQg{7be%pQ&$i2PEh_R|IX~6FQ0r8HcY~2Z#rb;` zg^kTT*&Eb&Je{IjHf zli@Go%89(d!+wSwneU>I!Dm5e`$qCTmT!q^vT9-H$kK3JjFB{lR}_eIUR)H@jx=a| zta^@2zj0Xb;lqdTl9PEToMz&D^X##PT6oOpQ!~?yVwa}+} z{tBwxU&_hk)7qS#-L!gpmmQg?{yN#`?&lUa@aL~0>M4U7w@OVcWbmtjjsMuoCMzWg z`IbUl^DAf1zlyhMf6?<#tls?;tz0JWF}ZYCN8d^m-7wa>ph>^UXCw3yTKIxJe4>z- z7L)h+Yi3eZkhOx|!zMC~EZh$n$N0q@(a2A+k#|1&ujRc6s1fSObynNFHh3! zTFjr0r@BG;X=s9;p}Sx$&EZn0%@qbLx&7p{PN8dzOp;(SYm^lMNeGz z?77hQiPG(?lZp)RHLACzP?yy=(SyoFT`t|p5@$is$dPwx(iuHVsBG01^#JIABVMXF zXke6A$QA4}^J=1`$bQ^uUPg58>$1b!Ffpk&Bv+;>0@h|hjfB*NM;{{J&l#`lSqtM0 zb;49CWZo%%|9-YJI3!?m33BdlFxLKv^ksz>zvetnja1km%5-&qJ3I54id& zqAuWbtn=c<&TgAkv7w%XNo^kLh&Dz6z000B`k(Kv{1i27x*!K6Bg}R8k}Zsz7nWjg zw3|My{x;v!tSC~rGyei2!5Nq~q|QRF&e+48mdTZ#<@zsvMfFnNy^Nh z2zB$)HEHF4#cQjjpK$CaioC+i;VBKR)UGjvX3_uO3N1gRl~w$Q>L~ZaVtk!GAXOz zjL)f3QoqmG6;eNc!nxO-heX4mp2nrXRg;aGyJ?&78)}dt>Up!EkB=|D*NXp8n{mUw zVe@kRWc|N8I}7K;i%GQwB{ErU{Dq4{tkyWgXfu= zrBOgi_AY*r;3a2wQa4D|7!3CTljKBWiM;QZ+Utj+XF2pvy%(!h=kLHQ=52G-ts|o3w}7 z2jl(pQ9-_ecfH)y66Tli@OpHXK;v7r)T=l8|6Ryd*qWFp-O7(`yE?C5^jM5=cQ542 zi|&G~H0>;On?vJgo(L`QWkbEm0ET*q)H@!}cN36ndhV}ZJ-%{f;#I$(_gr!5@#x3<6jOc#AYP+QG55tgOr*$HjTR zMp9P3WSq6`r@`2C-PJ_WCnRXNoA1#vKB<2j2!F>|;@U+p_O;EUD#*S=qx>b%H^Cv* z56)i{Q}(4bEcx9cpTo*n)pa^Ok;djh-M)%3pOx!7(kvYtG4$<+(|m#~V;e3TA`3Fp zAeJ86_;^poHnPH^;#3jpuBJ{Pk*sMa_ z>c!-2v+h9V@9QqA{V-bMPiMdUdOYJt-pR4<1Q@H zw^b--Y&Et^D{J=Lxh9;YI3lE}fXi&HFy;|F!lOzi>A1?4@|?@X&)*_m_1$d?xbv@2 zS#wMMrEnC&IAMzyItpXSNUa{UprZcq0Xb3s;sOFy7xbAKAot5-)-4fmifF; zW~pd}yPVMW;!O+->C?-W%*h5&4CCj>{v|6~PTr8V;`DTidT=3mgtVNI&XUPj5GR@3D#(ekl+zDETG4w1w0hj3? znwqBFZ;#=Cr~MeItPD<3UgQmIVBm z2T|y<3^w>yqQ~>2ZJ2M8O*G4f+M@4~g2I{WmSKWUkA=GTIwQVpZ8--D^M5dpGGkC8 z?eFiu<2Z7EDr>XdaM=^+dh*&>qf?5w zFYN(~(PJOy3A>~G!KVzy{EW3Xs-j8n!`QZ;N1cCVSmWCK;>_U|RoM)p$3Z6U{QVsT z7GHmT52&lV*m;sD3qi;o+0+lD>|F82HUk%Q5l-*mx5>09%L|LjQ^CCt*$3&GGcUj6 zGAO1)H5`bG2X1l1J(Xl2CI3+B`)^y-)FpKui?7z5Pg1j^92G5~`)8~ukm-T1gPj5& ziN-Op9$w2iGJMH4`C-ixH_VAG&KPbVHk^Df-_pq#7)X1LENl?59@Y_5&zAMJJ*vpH z+DgpjTG$jW?maouluG>LEvtV!>3>KX-4CU~{>avDT3fl^_ncQ-y?!HiZT{Lhiz%$= zn9GrSp3Dc!z!1uwXpT&^_Wzd%|GI!ldZ$G%V z1u=SS^B{Sa)_tX#U=kP*7moQ zVcnPSm8BiLVL>>2aITX{lFUvHK)ViBy5~?Y`a)a(L9ty5^Q5j%Itp5z!|UnjF!D@W z%*;@H2rC_-k3~Z*;6x^4*c;g?#2cA>f~}iZs+^AdWU?{xIk~?&a3xI+`f{WctH28 z!t_U>+?R`+5o_*u%8Oh5c&4&=EJmNV?qV34-kz^p7o;Fj(e-dmo5kq>O$j>tKgl!z*yc1nCmf~!j5{sJMvnitap?v)kk(2;% z(WU)~<0p(gC=@bSipOmWr+hKPy&~PL`t0`^Bh?5r)^U$r@G4Ho_Tv&gCM`;x_5TU* z0}uR-N5@W`D93UM&YrnBVbGmY>xG7PMECn?;^Z9|r{R;ckF->sQ~EJRva#%i2MI9osqtZW{IBd5;b!dn;7-*XURhKxc$dtJ5@ zV}E?Sdet(F92QPr1#FR?_A8Z;%9)-!KYRYM-kE>qq#rS-5m9SbCz9mYAa2 zkXdwNSTSU4%kf-^Hbkvy>%7cBm;U zKxws#-euhv6}kLwY?(ZN{q@(<67T#~$N1Rf+_`gtK;I3dg%5#%KtLcM5D*Cb4g`4F zrn+GyXpn2JGB`Yo|MJCBZ-rleknqpO!?xZ!qT@yx-7yw3PYksz~8%rk{f+l%K0R$eeeGiS6S3oGTOPE zl|s^vap)Y@l?FE9MrZ&2hnz$Gn{U2BL_~!2@51<)5imJ>_N*Y#fB$zu^iUum5D*9m z1pdMZunspnI|uK*_Z~8HZV(A%wdYN9Ark5D<%%btd1e>A7iQB1>(`s%}7A&lyJQ;p6QGKd%7A>kSL} z9mvc`K|T$bYDuTfQRmwkrIZFQs?xTdw=~)CcJ=>G=Z(p-@v+=jljP)NL7-(c#8x06 z5D*9m1Ox&C0fE4uhyV{{m_VN)sfw__76MF`j~t;@HtG5B7nzohO=yyVAkcrU=O test.pem + chmod 600 test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + euca-describe-instances + + ssh -i test.pem root@10.0.0.3 + +To see output from the various workers, switch screen windows with Ctrl+A " (quotation mark). + + .. image:: images/novascreens.png + From 17fd38e3cb277d51dcf9297178879a620623a855 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 23:46:18 +0000 Subject: [PATCH 073/119] Removing redundant check --- nova/auth/ldapdriver.py | 49 ++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index fa48c8435f..d54a0dfa65 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -135,34 +135,29 @@ class LdapDriver(object): if self.__ldap_user_exists(name): # Retrieve user by name user = self.__get_ldap_user(name) - if user.has_key('accessKey') and user.has_key('secretKey') \ - and user.has_key('isAdmin'): - raise exception.Duplicate("LDAP user %s already exists" \ - % name) + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ + [secret_key])) else: - # Entry could be malformed, test for missing attrs. - # Malformed entries are useless, replace attributes found. - attr = [] - if user.has_key('secretKey'): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ - [secret_key])) - else: - attr.append((self.ldap.MOD_ADD, 'secretKey', \ - [secret_key])) - if user.has_key('accessKey'): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ - [access_key])) - else: - attr.append((self.ldap.MOD_ADD, 'accessKey', \ - [access_key])) - if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ - [str(is_admin).upper()])) - else: - attr.append((self.ldap.MOD_ADD, 'isAdmin', \ - [str(is_admin).upper()])) - self.conn.modify_s(self.__uid_to_dn(name), attr) - return self.get_user(name) + attr.append((self.ldap.MOD_ADD, 'secretKey', \ + [secret_key])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ + [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', \ + [access_key])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ + [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, 'isAdmin', \ + [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) else: attr = [ ('objectclass', ['person', From 45324fc9f15135437051eaaedda68a5ef1f0da7a Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 23:53:01 +0000 Subject: [PATCH 074/119] Raising an exception if the user doesn't exist before trying to modify its attributes --- nova/auth/ldapdriver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index d54a0dfa65..5727c8da31 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -158,6 +158,8 @@ class LdapDriver(object): [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) + else: + raise exception.NotFound("User %s doesn't exist" % name) else: attr = [ ('objectclass', ['person', From abdb8080e365a584c64ce6562934eefb750568ba Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 00:08:47 +0000 Subject: [PATCH 075/119] Clarifying previously commited exception message --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 5727c8da31..45ea0683d2 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -159,7 +159,7 @@ class LdapDriver(object): self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: - raise exception.NotFound("User %s doesn't exist" % name) + raise exception.NotFound("LDAP object for %s doesn't exist" % name) else: attr = [ ('objectclass', ['person', From 70371ab447bff6af36f12ad9594eb6ffdbff4396 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 00:26:41 +0000 Subject: [PATCH 076/119] pep8 fix --- nova/auth/ldapdriver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 45ea0683d2..9baf45c924 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -159,7 +159,8 @@ class LdapDriver(object): self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: - raise exception.NotFound("LDAP object for %s doesn't exist" % name) + raise exception.NotFound("LDAP object for %s doesn't exist" + % name) else: attr = [ ('objectclass', ['person', From 9fdff2a0f0b45d7ddf1df58f83ac723fc8d99532 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 00:34:20 +0000 Subject: [PATCH 077/119] More pep8 fixes to remove deprecated functions --- nova/auth/ldapdriver.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 9baf45c924..c10939d743 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -138,19 +138,19 @@ class LdapDriver(object): # Entry could be malformed, test for missing attrs. # Malformed entries are useless, replace attributes found. attr = [] - if user.has_key('secretKey'): + if 'secretKey' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ [secret_key])) else: attr.append((self.ldap.MOD_ADD, 'secretKey', \ [secret_key])) - if user.has_key('accessKey'): + if 'accessKey' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ [access_key])) else: attr.append((self.ldap.MOD_ADD, 'accessKey', \ [access_key])) - if user.has_key('isAdmin'): + if 'isAdmin' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ [str(is_admin).upper()])) else: @@ -298,13 +298,13 @@ class LdapDriver(object): attr = [] # Retrieve user by name user = self.__get_ldap_user(uid) - if user.has_key('secretKey'): + if 'secretKey' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'secretKey', \ user['secretKey'])) - if user.has_key('accessKey'): + if 'accessKey' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'accessKey', \ user['accessKey'])) - if user.has_key('isAdmin'): + if 'isAdmin' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'isAdmin', \ user['isAdmin'])) self.conn.modify_s(self.__uid_to_dn(uid), attr) @@ -513,8 +513,8 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr is None: return None - if (attr.has_key('accessKey') and attr.has_key('secretKey') \ - and attr.has_key('isAdmin')): + if ('accessKey' in attr.keys() and 'secretKey' in attr.keys() \ + and 'isAdmin' in attr.keys()): return { 'id': attr['uid'][0], 'name': attr['cn'][0], From 180c94585ac9bb0e72a936f64ed27052af395999 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Wed, 8 Dec 2010 11:59:37 -0600 Subject: [PATCH 078/119] removing extraneous config ilnes --- doc/source/adminguide/multi.node.install.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst index 3b06d7d911..fcb76c5e53 100644 --- a/doc/source/adminguide/multi.node.install.rst +++ b/doc/source/adminguide/multi.node.install.rst @@ -89,9 +89,6 @@ Nova development has consolidated all .conf files to nova.conf as of November 20 --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 - --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 - --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 - #. Create a nova group:: sudo addgroup nova From d4b6cfe98f0ce81c21a45f420ce30c5c693c1144 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 8 Dec 2010 14:05:50 -0600 Subject: [PATCH 079/119] Got get_diagnostics in working order --- nova/virt/xenapi/vm_utils.py | 23 +++++++++++++++++++++++ nova/virt/xenapi/vmops.py | 28 ++-------------------------- nova/virt/xenapi_conn.py | 4 ++++ 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 801867bd41..c87a131a01 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -23,6 +23,7 @@ import logging import urllib from twisted.internet import defer +from xml.dom.minidom import parseString from nova import flags from nova import utils @@ -220,6 +221,28 @@ class VMHelper(): 'num_cpu': record['VCPUs_max'], 'cpu_time': 0} + @classmethod + def compile_diagnostics(cls, session, record): + try: + host = session.get_xenapi_host() + host_ip = session.get_xenapi().host.get_record(host)["address"] + metrics = session.get_xenapi().VM_guest_metrics.get_record( + record["guest_metrics"]) + diags = { + "Kernel": metrics["os_version"]["uname"], + "Distro": metrics["os_version"]["name"]} + xml = get_rrd(host_ip, record["uuid"]) + rrd = parseString(xml) + for i, node in enumerate(rrd.firstChild.childNodes): + # We don't want all of the extra garbage + if i >= 3 and i <= 11: + ref = node.childNodes + # Name and Value + diags[ref[0].firstChild.data] = ref[6].firstChild.data + return diags + except XenAPI.Failure as e: + return {"Unable to retrieve diagnostics": e} + def get_rrd(host, uuid): """Return the VM RRD XML as a string""" diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ba73079ec7..542d4894cd 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -21,7 +21,6 @@ Management class for VM-related functions (spawn, reboot, etc). import logging from twisted.internet import defer -from xml.dom.minidom import parseString from nova import db from nova import context @@ -131,35 +130,12 @@ class VMOps(object): return VMHelper.compile_info(rec) def get_diagnostics(self, instance_id): - """Return data about the VM diagnostics""" + """Return data about VM diagnostics""" vm = VMHelper.lookup_blocking(self._session, instance_id) if vm is None: raise Exception("instance not present %s" % instance_id) rec = self._session.get_xenapi().VM.get_record(vm) - try: - metrics = self._session.get_xenapi().VM_guest_metrics.get_record( - rec["guest_metrics"]) - diags = { - "Power State": rec["power_state"], - "Dom ID": rec["domid"], - "UUID": rec["uuid"], - "Kernel": metrics["os_version"]["uname"], - "Distro": metrics["os_version"]["name"]} - - xml = get_rrd(self._session.get_xenapi_host()["address"], - rec["uuid"]) - rrd = parseString(xml) - for i, node in enumerate(rrd.firstChild.childNodes): - # We don't want all of the extra garbage - if i >= 3 and i <= 11: - ref = node.childNodes - # Name and Value - diags[ref[0].firstChild.data] = ref[6].firstChild.data - - return {rec["name_label"]: diags} - except XenAPI.Failure as e: - return { - rec["name_label"]: "Unable to retrieve diagnostics: %s" % e} + return VMHelper.compile_diagnostics(self._session, rec) def get_console_output(self, instance): """ Return snapshot of console """ diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 26b30bf927..ac5f5e342d 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -126,6 +126,10 @@ class XenAPIConnection(object): """ Return data about VM instance """ return self._vmops.get_info(instance_id) + def get_diagnostics(self, instance_id): + """ Return data about VM diagnostics """ + return self._vmops.get_diagnostics(instance_id) + def get_console_output(self, instance): """ Return snapshot of console """ return self._vmops.get_console_output(instance) From fd7931847de7cb24c629380fb71bca7833710edc Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 8 Dec 2010 14:16:49 -0600 Subject: [PATCH 080/119] Fixed docstrings --- nova/virt/xenapi/vm_utils.py | 1 + nova/virt/xenapi_conn.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index c87a131a01..2a75f9dbf9 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -223,6 +223,7 @@ class VMHelper(): @classmethod def compile_diagnostics(cls, session, record): + """Compile VM diagnostics data""" try: host = session.get_xenapi_host() host_ip = session.get_xenapi().host.get_record(host)["address"] diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index ac5f5e342d..2153810c85 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -127,7 +127,7 @@ class XenAPIConnection(object): return self._vmops.get_info(instance_id) def get_diagnostics(self, instance_id): - """ Return data about VM diagnostics """ + """Return data about VM diagnostics""" return self._vmops.get_diagnostics(instance_id) def get_console_output(self, instance): From 708425aa5b42aae0f399b127ee5a648b7162b05e Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 8 Dec 2010 12:20:44 -0800 Subject: [PATCH 081/119] add bzr to the dev dependencies --- tools/pip-requires | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/pip-requires b/tools/pip-requires index 5480733265..17a1a4c5c1 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -20,3 +20,4 @@ mox==0.5.0 -f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz greenlet==0.3.1 nose +bzr From 0ae019062712fd15dd9e040a3fa60546db9c4111 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Wed, 8 Dec 2010 23:47:25 +0000 Subject: [PATCH 082/119] added to Authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 62f0c49d5d..a1703b279d 100644 --- a/Authors +++ b/Authors @@ -23,6 +23,7 @@ Rick Clark Ryan Lucio Soren Hansen Todd Willey +Trey Morris Vishvananda Ishaya Youcef Laribi Zhixue Wu From 3c85f1b7ed593a2d4d126a34241f217da5cf7ce6 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 8 Dec 2010 17:18:27 -0800 Subject: [PATCH 083/119] intermediate commit to checkpoint progress all relevant tests are passing except volume, next step is volume manager fixery --- nova/manager.py | 3 +- nova/objectstore/image.py | 3 + nova/rpc.py | 75 ++++++------------- nova/service.py | 1 - nova/test.py | 48 ++++++------- nova/tests/api/__init__.py | 3 +- nova/tests/auth_unittest.py | 5 +- nova/tests/cloud_unittest.py | 4 +- nova/tests/compute_unittest.py | 39 +++++----- nova/tests/rpc_unittest.py | 34 ++++----- nova/tests/service_unittest.py | 9 +-- nova/tests/virt_unittest.py | 7 +- nova/tests/volume_unittest.py | 56 +++++++-------- nova/utils.py | 23 +++--- nova/virt/images.py | 9 +-- nova/virt/libvirt_conn.py | 28 ++++---- nova/virt/xenapi/vm_utils.py | 28 +++----- nova/virt/xenapi_conn.py | 1 - nova/volume/driver.py | 127 ++++++++++++++------------------- run_tests.py | 14 +++- 20 files changed, 226 insertions(+), 291 deletions(-) diff --git a/nova/manager.py b/nova/manager.py index 5e067bd088..a343d7fc6a 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -55,7 +55,6 @@ from nova import utils from nova import flags from nova.db import base -from twisted.internet import defer FLAGS = flags.FLAGS @@ -69,7 +68,7 @@ class Manager(base.Base): def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval""" - yield + return def init_host(self): """Do any initialization that needs to be run if this is a standalone diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 7292dbab8f..2fe0b0117c 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -26,6 +26,7 @@ Requires decryption using keys in the manifest. import binascii import glob import json +import logging import os import shutil import tarfile @@ -264,6 +265,8 @@ class Image(object): if err: raise exception.Error("Failed to decrypt initialization " "vector: %s" % err) + logging.debug(iv) + _out, err = utils.execute( 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' % (encrypted_filename, key, iv, decrypted_filename), diff --git a/nova/rpc.py b/nova/rpc.py index 86a29574f3..652b9e4aa8 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -25,18 +25,18 @@ import json import logging import sys import time +import traceback import uuid from carrot import connection as carrot_connection from carrot import messaging from eventlet import greenthread -from twisted.internet import defer -from twisted.internet import task +from nova import context from nova import exception from nova import fakerabbit from nova import flags -from nova import context +from nova import utils FLAGS = flags.FLAGS @@ -128,17 +128,9 @@ class Consumer(messaging.Consumer): def attach_to_eventlet(self): """Only needed for unit tests!""" - def fetch_repeatedly(): - while True: - self.fetch(enable_callbacks=True) - greenthread.sleep(0.1) - greenthread.spawn(fetch_repeatedly) - - def attach_to_twisted(self): - """Attach a callback to twisted that fires 10 times a second""" - loop = task.LoopingCall(self.fetch, enable_callbacks=True) - loop.start(interval=0.1) - return loop + timer = utils.LoopingCall(self.fetch, enable_callbacks=True) + timer.start(0.1) + return timer class Publisher(messaging.Publisher): @@ -197,10 +189,13 @@ class AdapterConsumer(TopicConsumer): node_args = dict((str(k), v) for k, v in args.iteritems()) # NOTE(vish): magic is fun! # pylint: disable-msg=W0142 - d = defer.maybeDeferred(node_func, context=ctxt, **node_args) - if msg_id: - d.addCallback(lambda rval: msg_reply(msg_id, rval, None)) - d.addErrback(lambda e: msg_reply(msg_id, None, e)) + try: + rval = node_func(context=ctxt, **node_args) + if msg_id: + msg_reply(msg_id, rval, None) + except Exception as e: + if msg_id: + msg_reply(msg_id, None, sys.exc_info()) return @@ -244,11 +239,11 @@ def msg_reply(msg_id, reply=None, failure=None): failure should be a twisted failure object""" if failure: - message = failure.getErrorMessage() - traceback = failure.getTraceback() + message = str(failure[1]) + tb = traceback.format_exception(*failure) logging.error("Returning exception %s to caller", message) - logging.error(traceback) - failure = (failure.type.__name__, str(failure.value), traceback) + logging.error(tb) + failure = (failure[0].__name__, str(failure[1]), tb) conn = Connection.instance() publisher = DirectPublisher(connection=conn, msg_id=msg_id) try: @@ -313,8 +308,8 @@ def call(context, topic, msg): _pack_context(msg, context) class WaitMessage(object): - def __call__(self, data, message): + LOG.debug('data %s, msg %s', data, message) """Acks message and sets result.""" message.ack() if data['failure']: @@ -337,41 +332,11 @@ def call(context, topic, msg): except StopIteration: pass consumer.close() + if isinstance(wait_msg.result, Exception): + raise wait_msg.result return wait_msg.result -def call_twisted(context, topic, msg): - """Sends a message on a topic and wait for a response""" - LOG.debug("Making asynchronous call...") - msg_id = uuid.uuid4().hex - msg.update({'_msg_id': msg_id}) - LOG.debug("MSG_ID is %s" % (msg_id)) - _pack_context(msg, context) - - conn = Connection.instance() - d = defer.Deferred() - consumer = DirectConsumer(connection=conn, msg_id=msg_id) - - def deferred_receive(data, message): - """Acks message and callbacks or errbacks""" - message.ack() - if data['failure']: - return d.errback(RemoteError(*data['failure'])) - else: - return d.callback(data['result']) - - consumer.register_callback(deferred_receive) - injected = consumer.attach_to_twisted() - - # clean up after the injected listened and return x - d.addCallback(lambda x: injected.stop() and x or x) - - publisher = TopicPublisher(connection=conn, topic=topic) - publisher.send(msg) - publisher.close() - return d - - def cast(context, topic, msg): """Sends a message on a topic without waiting for a response""" LOG.debug("Making asynchronous cast...") diff --git a/nova/service.py b/nova/service.py index 9454d4049b..55a0bb2120 100644 --- a/nova/service.py +++ b/nova/service.py @@ -160,7 +160,6 @@ class Service(object, service.Service): except exception.NotFound: logging.warn("Service killed that has no database entry") - @defer.inlineCallbacks def periodic_tasks(self): """Tasks to be run at a periodic interval""" yield self.manager.periodic_tasks(context.get_admin_context()) diff --git a/nova/test.py b/nova/test.py index 5c2a728196..bbf063aca3 100644 --- a/nova/test.py +++ b/nova/test.py @@ -25,11 +25,11 @@ and some black magic for inline callbacks. import datetime import sys import time +import unittest import mox import stubout from twisted.internet import defer -from twisted.trial import unittest from nova import context from nova import db @@ -94,7 +94,7 @@ class TrialTestCase(unittest.TestCase): db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, self.start) db.network_disassociate_all(ctxt) - rpc.Consumer.attach_to_twisted = self.originalAttach + rpc.Consumer.attach_to_eventlet = self.originalAttach for x in self.injected: try: x.stop() @@ -125,31 +125,31 @@ class TrialTestCase(unittest.TestCase): for k, v in self._original_flags.iteritems(): setattr(FLAGS, k, v) - def run(self, result=None): - test_method = getattr(self, self._testMethodName) - setattr(self, - self._testMethodName, - self._maybeInlineCallbacks(test_method, result)) - rv = super(TrialTestCase, self).run(result) - setattr(self, self._testMethodName, test_method) - return rv + #def run(self, result=None): + # test_method = getattr(self, self._testMethodName) + # setattr(self, + # self._testMethodName, + # self._maybeInlineCallbacks(test_method, result)) + # rv = super(TrialTestCase, self).run(result) + # setattr(self, self._testMethodName, test_method) + # return rv - def _maybeInlineCallbacks(self, func, result): - def _wrapped(): - g = func() - if isinstance(g, defer.Deferred): - return g - if not hasattr(g, 'send'): - return defer.succeed(g) + #def _maybeInlineCallbacks(self, func, result): + # def _wrapped(): + # g = func() + # if isinstance(g, defer.Deferred): + # return g + # if not hasattr(g, 'send'): + # return defer.succeed(g) - inlined = defer.inlineCallbacks(func) - d = inlined() - return d - _wrapped.func_name = func.func_name - return _wrapped + # inlined = defer.inlineCallbacks(func) + # d = inlined() + # return d + # _wrapped.func_name = func.func_name + # return _wrapped def _monkey_patch_attach(self): - self.originalAttach = rpc.Consumer.attach_to_twisted + self.originalAttach = rpc.Consumer.attach_to_eventlet def _wrapped(innerSelf): rv = self.originalAttach(innerSelf) @@ -157,4 +157,4 @@ class TrialTestCase(unittest.TestCase): return rv _wrapped.func_name = self.originalAttach.func_name - rpc.Consumer.attach_to_twisted = _wrapped + rpc.Consumer.attach_to_eventlet = _wrapped diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index 9caa8c9d0d..cdc1bbf001 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -78,4 +78,5 @@ class Test(unittest.TestCase): if __name__ == '__main__': - unittest.main() + pass + #unittest.main() diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index fe891beeeb..129ff223d5 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -16,10 +16,13 @@ # License for the specific language governing permissions and limitations # under the License. -import logging +#import logging from M2Crypto import X509 import unittest +import eventlet +logging = eventlet.import_patched('logging') + from nova import crypto from nova import flags from nova import test diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 9886a2449b..b7b856da52 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -27,8 +27,6 @@ import tempfile import time from eventlet import greenthread -from twisted.internet import defer -import unittest from xml.etree import ElementTree from nova import context @@ -186,7 +184,7 @@ class CloudTestCase(test.TrialTestCase): logging.debug("Need to watch instance %s until it's running..." % instance['instance_id']) while True: - rv = yield defer.succeed(time.sleep(1)) + greenthread.sleep(1) info = self.cloud._get_instance(instance['instance_id']) logging.debug(info['state']) if info['state'] == power_state.RUNNING: diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 6f3ef96cbb..67cea72c95 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -22,8 +22,6 @@ Tests For Compute import datetime import logging -from twisted.internet import defer - from nova import context from nova import db from nova import exception @@ -33,6 +31,7 @@ from nova import utils from nova.auth import manager from nova.compute import api as compute_api + FLAGS = flags.FLAGS @@ -94,24 +93,22 @@ class ComputeTestCase(test.TrialTestCase): db.security_group_destroy(self.context, group['id']) db.instance_destroy(self.context, ref[0]['id']) - @defer.inlineCallbacks def test_run_terminate(self): """Make sure it is possible to run and terminate instance""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) logging.info("Running instances: %s", instances) self.assertEqual(len(instances), 1) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) logging.info("After terminating instances: %s", instances) self.assertEqual(len(instances), 0) - @defer.inlineCallbacks def test_run_terminate_timestamps(self): """Make sure timestamps are set for launched and destroyed""" instance_id = self._create_instance() @@ -119,42 +116,40 @@ class ComputeTestCase(test.TrialTestCase): self.assertEqual(instance_ref['launched_at'], None) self.assertEqual(instance_ref['deleted_at'], None) launch = datetime.datetime.utcnow() - yield self.compute.run_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] > launch) self.assertEqual(instance_ref['deleted_at'], None) terminate = datetime.datetime.utcnow() - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) self.context = self.context.elevated(True) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] < terminate) self.assert_(instance_ref['deleted_at'] > terminate) - @defer.inlineCallbacks def test_reboot(self): """Ensure instance can be rebooted""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) - yield self.compute.reboot_instance(self.context, instance_id) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) + self.compute.reboot_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) - @defer.inlineCallbacks def test_console_output(self): """Make sure we can get console output from instance""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) - console = yield self.compute.get_console_output(self.context, + console = self.compute.get_console_output(self.context, instance_id) self.assert_(console) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) - @defer.inlineCallbacks def test_run_instance_existing(self): """Ensure failure when running an instance that already exists""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) - self.assertFailure(self.compute.run_instance(self.context, - instance_id), - exception.Error) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) + self.assertRaises(exception.Error, + self.compute.run_instance, + self.context, + instance_id) + self.compute.terminate_instance(self.context, instance_id) diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index f35b65a39c..c2ad5cd795 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -20,8 +20,6 @@ Unit Tests for remote procedure calls using queue """ import logging -from twisted.internet import defer - from nova import context from nova import flags from nova import rpc @@ -40,23 +38,22 @@ class RpcTestCase(test.TrialTestCase): self.consumer = rpc.AdapterConsumer(connection=self.conn, topic='test', proxy=self.receiver) - self.consumer.attach_to_twisted() + self.consumer.attach_to_eventlet() self.context = context.get_admin_context() def test_call_succeed(self): """Get a value through rpc call""" value = 42 - result = yield rpc.call_twisted(self.context, - 'test', {"method": "echo", + result = rpc.call(self.context, 'test', {"method": "echo", "args": {"value": value}}) self.assertEqual(value, result) def test_context_passed(self): """Makes sure a context is passed through rpc call""" value = 42 - result = yield rpc.call_twisted(self.context, - 'test', {"method": "context", - "args": {"value": value}}) + result = rpc.call(self.context, + 'test', {"method": "context", + "args": {"value": value}}) self.assertEqual(self.context.to_dict(), result) def test_call_exception(self): @@ -67,14 +64,17 @@ class RpcTestCase(test.TrialTestCase): to an int in the test. """ value = 42 - self.assertFailure(rpc.call_twisted(self.context, 'test', - {"method": "fail", - "args": {"value": value}}), - rpc.RemoteError) + self.assertRaises(rpc.RemoteError, + rpc.call, + self.context, + 'test', + {"method": "fail", + "args": {"value": value}}) try: - yield rpc.call_twisted(self.context, - 'test', {"method": "fail", - "args": {"value": value}}) + rpc.call(self.context, + 'test', + {"method": "fail", + "args": {"value": value}}) self.fail("should have thrown rpc.RemoteError") except rpc.RemoteError as exc: self.assertEqual(int(exc.value), value) @@ -89,13 +89,13 @@ class TestReceiver(object): def echo(context, value): """Simply returns whatever value is sent in""" logging.debug("Received %s", value) - return defer.succeed(value) + return value @staticmethod def context(context, value): """Returns dictionary version of context""" logging.debug("Received %s", context) - return defer.succeed(context.to_dict()) + return context.to_dict() @staticmethod def fail(context, value): diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index a268bc4fe4..4f8d2d550f 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -143,7 +143,6 @@ class ServiceTestCase(test.TrialTestCase): # whether it is disconnected, it looks for a variable on itself called # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage - @defer.inlineCallbacks def test_report_state_no_service(self): host = 'foo' binary = 'bar' @@ -174,9 +173,8 @@ class ServiceTestCase(test.TrialTestCase): topic, 'nova.tests.service_unittest.FakeManager') serv.startService() - yield serv.report_state() + serv.report_state() - @defer.inlineCallbacks def test_report_state_newly_disconnected(self): host = 'foo' binary = 'bar' @@ -205,10 +203,9 @@ class ServiceTestCase(test.TrialTestCase): topic, 'nova.tests.service_unittest.FakeManager') serv.startService() - yield serv.report_state() + serv.report_state() self.assert_(serv.model_disconnected) - @defer.inlineCallbacks def test_report_state_newly_connected(self): host = 'foo' binary = 'bar' @@ -240,6 +237,6 @@ class ServiceTestCase(test.TrialTestCase): 'nova.tests.service_unittest.FakeManager') serv.startService() serv.model_disconnected = True - yield serv.report_state() + serv.report_state() self.assert_(not serv.model_disconnected) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index d49383fb7b..a4a8d3acf8 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -235,7 +235,7 @@ class NWFilterTestCase(test.TrialTestCase): 'project_id': 'fake'}) inst_id = instance_ref['id'] - def _ensure_all_called(_): + def _ensure_all_called(): instance_filter = 'nova-instance-%s' % instance_ref['name'] secgroup_filter = 'nova-secgroup-%s' % self.security_group['id'] for required in [secgroup_filter, 'allow-dhcp-server', @@ -253,7 +253,6 @@ class NWFilterTestCase(test.TrialTestCase): instance = db.instance_get(self.context, inst_id) d = self.fw.setup_nwfilters_for_instance(instance) - d.addCallback(_ensure_all_called) - d.addCallback(lambda _: self.teardown_security_group()) - + _ensure_all_called() + self.teardown_security_group() return d diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 12321a96fb..93d2ceab79 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -21,8 +21,6 @@ Tests for Volume Code. """ import logging -from twisted.internet import defer - from nova import context from nova import exception from nova import db @@ -56,51 +54,48 @@ class VolumeTestCase(test.TrialTestCase): vol['attach_status'] = "detached" return db.volume_create(context.get_admin_context(), vol)['id'] - @defer.inlineCallbacks def test_create_delete_volume(self): """Test volume can be created and deleted.""" volume_id = self._create_volume() - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), volume_id).id) - yield self.volume.delete_volume(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.NotFound, db.volume_get, self.context, volume_id) - @defer.inlineCallbacks def test_too_big_volume(self): """Ensure failure if a too large of a volume is requested.""" # FIXME(vish): validation needs to move into the data layer in # volume_create - defer.returnValue(True) + return True try: volume_id = self._create_volume('1001') - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) self.fail("Should have thrown TypeError") except TypeError: pass - @defer.inlineCallbacks def test_too_many_volumes(self): """Ensure that NoMoreTargets is raised when we run out of volumes.""" vols = [] total_slots = FLAGS.iscsi_num_targets for _index in xrange(total_slots): volume_id = self._create_volume() - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) vols.append(volume_id) volume_id = self._create_volume() - self.assertFailure(self.volume.create_volume(self.context, - volume_id), - db.NoMoreTargets) + self.assertRaises(db.NoMoreTargets, + self.volume.create_volume, + self.context, + volume_id) db.volume_destroy(context.get_admin_context(), volume_id) for volume_id in vols: - yield self.volume.delete_volume(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) - @defer.inlineCallbacks def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance.""" inst = {} @@ -115,15 +110,15 @@ class VolumeTestCase(test.TrialTestCase): instance_id = db.instance_create(self.context, inst)['id'] mountpoint = "/dev/sdf" volume_id = self._create_volume() - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) if FLAGS.fake_tests: db.volume_attached(self.context, volume_id, instance_id, mountpoint) else: - yield self.compute.attach_volume(self.context, - instance_id, - volume_id, - mountpoint) + self.compute.attach_volume(self.context, + instance_id, + volume_id, + mountpoint) vol = db.volume_get(context.get_admin_context(), volume_id) self.assertEqual(vol['status'], "in-use") self.assertEqual(vol['attach_status'], "attached") @@ -131,25 +126,26 @@ class VolumeTestCase(test.TrialTestCase): instance_ref = db.volume_get_instance(self.context, volume_id) self.assertEqual(instance_ref['id'], instance_id) - self.assertFailure(self.volume.delete_volume(self.context, volume_id), - exception.Error) + self.assertRaises(exception.Error, + self.volume.delete_volume, + self.context, + volume_id) if FLAGS.fake_tests: db.volume_detached(self.context, volume_id) else: - yield self.compute.detach_volume(self.context, - instance_id, - volume_id) + self.compute.detach_volume(self.context, + instance_id, + volume_id) vol = db.volume_get(self.context, volume_id) self.assertEqual(vol['status'], "available") - yield self.volume.delete_volume(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.Error, db.volume_get, self.context, volume_id) db.instance_destroy(self.context, instance_id) - @defer.inlineCallbacks def test_concurrent_volumes_get_different_targets(self): """Ensure multiple concurrent volumes get different targets.""" volume_ids = [] @@ -164,15 +160,11 @@ class VolumeTestCase(test.TrialTestCase): self.assert_(iscsi_target not in targets) targets.append(iscsi_target) logging.debug("Target %s allocated", iscsi_target) - deferreds = [] total_slots = FLAGS.iscsi_num_targets for _index in xrange(total_slots): volume_id = self._create_volume() d = self.volume.create_volume(self.context, volume_id) - d.addCallback(_check) - d.addErrback(self.fail) - deferreds.append(d) - yield defer.DeferredList(deferreds) + _check(d) for volume_id in volume_ids: self.volume.delete_volume(self.context, volume_id) diff --git a/nova/utils.py b/nova/utils.py index 66047ae8bc..2c43203d88 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -34,8 +34,6 @@ from xml.sax import saxutils from eventlet import event from eventlet import greenthread -from twisted.internet.threads import deferToThread - from nova import exception from nova import flags from nova.exception import ProcessExecutionError @@ -78,7 +76,7 @@ def fetchfile(url, target): def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - logging.debug("Running cmd: %s", cmd) + logging.debug("Running cmd (subprocess): %s", cmd) env = os.environ.copy() if addl_env: env.update(addl_env) @@ -98,6 +96,10 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): stdout=stdout, stderr=stderr, cmd=cmd) + # NOTE(termie): this appears to be necessary to let the subprocess call + # clean something up in between calls, without it two + # execute calls in a row hangs the second one + greenthread.sleep(0) return result @@ -126,13 +128,14 @@ def debug(arg): def runthis(prompt, cmd, check_exit_code=True): logging.debug("Running %s" % (cmd)) - exit_code = subprocess.call(cmd.split(" ")) - logging.debug(prompt % (exit_code)) - if check_exit_code and exit_code != 0: - raise ProcessExecutionError(exit_code=exit_code, - stdout=None, - stderr=None, - cmd=cmd) + rv, err = execute(cmd, check_exit_code=check_exit_code) + #exit_code = subprocess.call(cmd.split(" ")) + #logging.debug(prompt % (exit_code)) + #if check_exit_code and exit_code != 0: + # raise ProcessExecutionError(exit_code=exit_code, + # stdout=None, + # stderr=None, + # cmd=cmd) def generate_uid(topic, size=8): diff --git a/nova/virt/images.py b/nova/virt/images.py index 981aa5cf36..4d7c65f126 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -26,7 +26,7 @@ import time import urlparse from nova import flags -from nova import process +from nova import utils from nova.auth import manager from nova.auth import signer from nova.objectstore import image @@ -63,15 +63,16 @@ def _fetch_s3_image(image, path, user, project): cmd = ['/usr/bin/curl', '--fail', '--silent', url] for (k, v) in headers.iteritems(): - cmd += ['-H', '%s: %s' % (k, v)] + cmd += ['-H', '"%s: %s"' % (k, v)] cmd += ['-o', path] - return process.SharedPool().execute(executable=cmd[0], args=cmd[1:]) + cmd_out = ' '.join(cmd) + return utils.execute(cmd_out) def _fetch_local_image(image, path, user, project): source = _image_path('%s/image' % image) - return process.simple_execute('cp %s %s' % (source, path)) + return utils.execute('cp %s %s' % (source, path)) def _image_path(path): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c09a7c01dc..715e7234cf 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -54,7 +54,6 @@ from nova import context from nova import db from nova import exception from nova import flags -from nova import process from nova import utils #from nova.api import context from nova.auth import manager @@ -366,8 +365,8 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): logging.info('cool, it\'s a device') - r = process.simple_execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) + r = utils.execute("sudo dd if=%s iflag=nonblock" % + virsh_output, check_exit_code=False) return r[0] else: return '' @@ -389,13 +388,13 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - process.simple_execute('sudo chown %d %s' % (os.getuid(), - console_log)) + utils.execute('sudo chown %d %s' % (os.getuid(), + console_log)) if FLAGS.libvirt_type == 'xen': # Xen is special - virsh_output = process.simple_execute("virsh ttyconsole %s" % - instance['name']) + virsh_output = utils.execute("virsh ttyconsole %s" % + instance['name']) data = self._flush_xen_console(virsh_output) fpath = self._append_to_file(data, console_log) else: @@ -411,8 +410,8 @@ class LibvirtConnection(object): prefix + fname) # ensure directories exist and are writable - process.simple_execute('mkdir -p %s' % basepath(prefix='')) - process.simple_execute('chmod 0777 %s' % basepath(prefix='')) + utils.execute('mkdir -p %s' % basepath(prefix='')) + utils.execute('chmod 0777 %s' % basepath(prefix='')) # TODO(termie): these are blocking calls, it would be great # if they weren't. @@ -443,9 +442,9 @@ class LibvirtConnection(object): project) def execute(cmd, process_input=None, check_exit_code=True): - return process.simple_execute(cmd=cmd, - process_input=process_input, - check_exit_code=check_exit_code) + return utils.execute(cmd=cmd, + process_input=process_input, + check_exit_code=check_exit_code) key = str(inst['key_data']) net = None @@ -471,7 +470,7 @@ class LibvirtConnection(object): execute=execute) if os.path.exists(basepath('disk')): - process.simple_execute('rm -f %s' % basepath('disk')) + utils.execute('rm -f %s' % basepath('disk')) local_bytes = (instance_types.INSTANCE_TYPES[inst.instance_type] ['local_gb'] @@ -485,8 +484,7 @@ class LibvirtConnection(object): local_bytes, resize, execute=execute) if FLAGS.libvirt_type == 'uml': - process.simple_execute('sudo chown root %s' % - basepath('disk')) + utils.execute('sudo chown root %s' % basepath('disk')) def to_xml(self, instance, rescue=False): # TODO(termie): cache? diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 99d484ca2b..b72b8e13d3 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -21,14 +21,13 @@ their attributes like VDIs, VIFs, as well as their lookup functions. import logging -from twisted.internet import defer - from nova import utils from nova.auth.manager import AuthManager from nova.compute import instance_types from nova.compute import power_state from nova.virt import images + XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, 'Running': power_state.RUNNING, @@ -36,6 +35,7 @@ XENAPI_POWER_STATE = { 'Suspended': power_state.SHUTDOWN, # FIXME 'Crashed': power_state.CRASHED} + XenAPI = None @@ -49,7 +49,6 @@ class VMHelper(): XenAPI = __import__('XenAPI') @classmethod - @defer.inlineCallbacks def create_vm(cls, session, instance, kernel, ramdisk): """Create a VM record. Returns a Deferred that gives the new VM reference.""" @@ -87,12 +86,11 @@ class VMHelper(): 'other_config': {}, } logging.debug('Created VM %s...', instance.name) - vm_ref = yield session.call_xenapi('VM.create', rec) + vm_ref = session.call_xenapi('VM.create', rec) logging.debug('Created VM %s as %s.', instance.name, vm_ref) - defer.returnValue(vm_ref) + return vm_ref @classmethod - @defer.inlineCallbacks def create_vbd(cls, session, vm_ref, vdi_ref, userdevice, bootable): """Create a VBD record. Returns a Deferred that gives the new VBD reference.""" @@ -111,13 +109,12 @@ class VMHelper(): vbd_rec['qos_algorithm_params'] = {} vbd_rec['qos_supported_algorithms'] = [] logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) - vbd_ref = yield session.call_xenapi('VBD.create', vbd_rec) + vbd_ref = session.call_xenapi('VBD.create', vbd_rec) logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, vdi_ref) - defer.returnValue(vbd_ref) + return vbd_ref @classmethod - @defer.inlineCallbacks def create_vif(cls, session, vm_ref, network_ref, mac_address): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" @@ -133,13 +130,12 @@ class VMHelper(): vif_rec['qos_algorithm_params'] = {} logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, network_ref) - vif_ref = yield session.call_xenapi('VIF.create', vif_rec) + vif_ref = session.call_xenapi('VIF.create', vif_rec) logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, vm_ref, network_ref) - defer.returnValue(vif_ref) + return vif_ref @classmethod - @defer.inlineCallbacks def fetch_image(cls, session, image, user, project, use_sr): """use_sr: True to put the image as a VDI in an SR, False to place it on dom0's filesystem. The former is for VM disks, the latter for @@ -156,12 +152,11 @@ class VMHelper(): args['password'] = user.secret if use_sr: args['add_partition'] = 'true' - task = yield session.async_call_plugin('objectstore', fn, args) - uuid = yield session.wait_for_task(task) - defer.returnValue(uuid) + task = session.async_call_plugin('objectstore', fn, args) + uuid = session.wait_for_task(task) + return uuid @classmethod - @utils.deferredToThread def lookup(cls, session, i): """ Look the instance i up, and returns it if available """ return VMHelper.lookup_blocking(session, i) @@ -179,7 +174,6 @@ class VMHelper(): return vms[0] @classmethod - @utils.deferredToThread def lookup_vm_vdis(cls, session, vm): """ Look for the VDIs that are attached to the VM """ return VMHelper.lookup_vm_vdis_blocking(session, vm) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index dacf9fe2b4..96d211cc08 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -167,7 +167,6 @@ class XenAPISession(object): self.get_xenapi_host(), plugin, fn, args) def wait_for_task(self, task): ->>>>>>> MERGE-SOURCE """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 156aad2a09..f675c91320 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -23,8 +23,6 @@ Drivers for volumes. import logging import os -from twisted.internet import defer - from nova import exception from nova import flags from nova import process @@ -55,14 +53,13 @@ flags.DEFINE_string('iscsi_ip_prefix', '127.0', class VolumeDriver(object): """Executes commands relating to Volumes.""" - def __init__(self, execute=process.simple_execute, + def __init__(self, execute=utils.execute, sync_exec=utils.execute, *args, **kwargs): # NOTE(vish): db is set by Manager self.db = None self._execute = execute self._sync_exec = sync_exec - @defer.inlineCallbacks def _try_execute(self, command): # NOTE(vish): Volume commands can partially fail due to timing, but # running them a second time on failure will usually @@ -70,15 +67,15 @@ class VolumeDriver(object): tries = 0 while True: try: - yield self._execute(command) - defer.returnValue(True) + self._execute(command) + return True except exception.ProcessExecutionError: tries = tries + 1 if tries >= FLAGS.num_shell_tries: raise logging.exception("Recovering from a failed execute." "Try number %s", tries) - yield self._execute("sleep %s" % tries ** 2) + self._execute("sleep %s" % tries ** 2) def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" @@ -86,53 +83,46 @@ class VolumeDriver(object): raise exception.Error("volume group %s doesn't exist" % FLAGS.volume_group) - @defer.inlineCallbacks def create_volume(self, volume): """Creates a logical volume.""" if int(volume['size']) == 0: sizestr = '100M' else: sizestr = '%sG' % volume['size'] - yield self._try_execute("sudo lvcreate -L %s -n %s %s" % - (sizestr, + self._try_execute("sudo lvcreate -L %s -n %s %s" % + (sizestr, volume['name'], FLAGS.volume_group)) - @defer.inlineCallbacks def delete_volume(self, volume): """Deletes a logical volume.""" - yield self._try_execute("sudo lvremove -f %s/%s" % - (FLAGS.volume_group, + self._try_execute("sudo lvremove -f %s/%s" % + (FLAGS.volume_group, volume['name'])) - @defer.inlineCallbacks def local_path(self, volume): - yield # NOTE(vish): stops deprecation warning + # NOTE(vish): stops deprecation warning escaped_group = FLAGS.volume_group.replace('-', '--') escaped_name = volume['name'].replace('-', '--') - defer.returnValue("/dev/mapper/%s-%s" % (escaped_group, - escaped_name)) + return "/dev/mapper/%s-%s" % (escaped_group, + escaped_name) def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" raise NotImplementedError() - @defer.inlineCallbacks def create_export(self, context, volume): """Exports the volume.""" raise NotImplementedError() - @defer.inlineCallbacks def remove_export(self, context, volume): """Removes an export for a logical volume.""" raise NotImplementedError() - @defer.inlineCallbacks def discover_volume(self, volume): """Discover volume on a remote host.""" raise NotImplementedError() - @defer.inlineCallbacks def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" raise NotImplementedError() @@ -155,14 +145,13 @@ class AOEDriver(VolumeDriver): dev = {'shelf_id': shelf_id, 'blade_id': blade_id} self.db.export_device_create_safe(context, dev) - @defer.inlineCallbacks def create_export(self, context, volume): """Creates an export for a logical volume.""" self._ensure_blades(context) (shelf_id, blade_id) = self.db.volume_allocate_shelf_and_blade(context, volume['id']) - yield self._try_execute( + self._try_execute( "sudo vblade-persist setup %s %s %s /dev/%s/%s" % (shelf_id, blade_id, @@ -176,33 +165,30 @@ class AOEDriver(VolumeDriver): # still works for the other volumes, so we # just wait a bit for the current volume to # be ready and ignore any errors. - yield self._execute("sleep 2") - yield self._execute("sudo vblade-persist auto all", - check_exit_code=False) - yield self._execute("sudo vblade-persist start all", - check_exit_code=False) + self._execute("sleep 2") + self._execute("sudo vblade-persist auto all", + check_exit_code=False) + self._execute("sudo vblade-persist start all", + check_exit_code=False) - @defer.inlineCallbacks def remove_export(self, context, volume): """Removes an export for a logical volume.""" (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) - yield self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - yield self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + self._try_execute("sudo vblade-persist stop %s %s" % + (shelf_id, blade_id)) + self._try_execute("sudo vblade-persist destroy %s %s" % + (shelf_id, blade_id)) - @defer.inlineCallbacks def discover_volume(self, _volume): """Discover volume on a remote host.""" - yield self._execute("sudo aoe-discover") - yield self._execute("sudo aoe-stat", check_exit_code=False) + self._execute("sudo aoe-discover") + self._execute("sudo aoe-stat", check_exit_code=False) - @defer.inlineCallbacks def undiscover_volume(self, _volume): """Undiscover volume on a remote host.""" - yield + pass class FakeAOEDriver(AOEDriver): @@ -252,7 +238,6 @@ class ISCSIDriver(VolumeDriver): target = {'host': host, 'target_num': target_num} self.db.iscsi_target_create_safe(context, target) - @defer.inlineCallbacks def create_export(self, context, volume): """Creates an export for a logical volume.""" self._ensure_iscsi_targets(context, volume['host']) @@ -261,61 +246,57 @@ class ISCSIDriver(VolumeDriver): volume['host']) iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - yield self._execute("sudo ietadm --op new " - "--tid=%s --params Name=%s" % - (iscsi_target, iscsi_name)) - yield self._execute("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path)) + self._execute("sudo ietadm --op new " + "--tid=%s --params Name=%s" % + (iscsi_target, iscsi_name)) + self._execute("sudo ietadm --op new --tid=%s " + "--lun=0 --params Path=%s,Type=fileio" % + (iscsi_target, volume_path)) - @defer.inlineCallbacks def remove_export(self, context, volume): """Removes an export for a logical volume.""" iscsi_target = self.db.volume_get_iscsi_target_num(context, volume['id']) - yield self._execute("sudo ietadm --op delete --tid=%s " - "--lun=0" % iscsi_target) - yield self._execute("sudo ietadm --op delete --tid=%s" % - iscsi_target) + self._execute("sudo ietadm --op delete --tid=%s " + "--lun=0" % iscsi_target) + self._execute("sudo ietadm --op delete --tid=%s" % + iscsi_target) - @defer.inlineCallbacks def _get_name_and_portal(self, volume_name, host): """Gets iscsi name and portal from volume name and host.""" - (out, _err) = yield self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % host) + (out, _err) = self._execute("sudo iscsiadm -m discovery -t " + "sendtargets -p %s" % host) for target in out.splitlines(): if FLAGS.iscsi_ip_prefix in target and volume_name in target: (location, _sep, iscsi_name) = target.partition(" ") break iscsi_portal = location.split(",")[0] - defer.returnValue((iscsi_name, iscsi_portal)) + return (iscsi_name, iscsi_portal) - @defer.inlineCallbacks def discover_volume(self, volume): """Discover volume on a remote host.""" (iscsi_name, - iscsi_portal) = yield self._get_name_and_portal(volume['name'], - volume['host']) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --login" % - (iscsi_name, iscsi_portal)) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v automatic" % - (iscsi_name, iscsi_portal)) - defer.returnValue("/dev/iscsi/%s" % volume['name']) + iscsi_portal) = self._get_name_and_portal(volume['name'], + volume['host']) + self._execute("sudo iscsiadm -m node -T %s -p %s --login" % + (iscsi_name, iscsi_portal)) + self._execute("sudo iscsiadm -m node -T %s -p %s --op update " + "-n node.startup -v automatic" % + (iscsi_name, iscsi_portal)) + return "/dev/iscsi/%s" % volume['name'] - @defer.inlineCallbacks def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" (iscsi_name, - iscsi_portal) = yield self._get_name_and_portal(volume['name'], - volume['host']) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v manual" % - (iscsi_name, iscsi_portal)) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % - (iscsi_name, iscsi_portal)) - yield self._execute("sudo iscsiadm -m node --op delete " - "--targetname %s" % iscsi_name) + iscsi_portal) = self._get_name_and_portal(volume['name'], + volume['host']) + self._execute("sudo iscsiadm -m node -T %s -p %s --op update " + "-n node.startup -v manual" % + (iscsi_name, iscsi_portal)) + self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % + (iscsi_name, iscsi_portal)) + self._execute("sudo iscsiadm -m node --op delete " + "--targetname %s" % iscsi_name) class FakeISCSIDriver(ISCSIDriver): diff --git a/run_tests.py b/run_tests.py index 3d427d8af2..883d2b768c 100644 --- a/run_tests.py +++ b/run_tests.py @@ -39,11 +39,16 @@ Due to our use of multiprocessing it we frequently get some ignorable """ +import eventlet +eventlet.monkey_patch() + import __main__ import os import sys + from twisted.scripts import trial as trial_script +import unittest from nova import flags from nova import twistd @@ -56,12 +61,12 @@ from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * from nova.tests.misc_unittest import * from nova.tests.network_unittest import * -from nova.tests.objectstore_unittest import * -from nova.tests.process_unittest import * +#from nova.tests.objectstore_unittest import * +#from nova.tests.process_unittest import * from nova.tests.quota_unittest import * from nova.tests.rpc_unittest import * from nova.tests.scheduler_unittest import * -from nova.tests.service_unittest import * +#from nova.tests.service_unittest import * from nova.tests.twistd_unittest import * from nova.tests.validator_unittest import * from nova.tests.virt_unittest import * @@ -82,6 +87,8 @@ if __name__ == '__main__': config = OptionsClass() argv = config.parseOptions() + argv = FLAGS(sys.argv) + FLAGS.verbose = True # TODO(termie): these should make a call instead of doing work on import @@ -90,6 +97,7 @@ if __name__ == '__main__': else: from nova.tests.real_flags import * + # Establish redirect for STDERR sys.stderr.flush() err = open(FLAGS.tests_stderr, 'w+', 0) From 653373842815a1ba9992d3d662431ba102ac8ce1 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 8 Dec 2010 17:21:43 -0800 Subject: [PATCH 084/119] port volume manager to eventlet also --- nova/volume/manager.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 589e7d7d97..7da125cac0 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -45,7 +45,6 @@ intact. import logging import datetime -from twisted.internet import defer from nova import context from nova import exception @@ -86,7 +85,6 @@ class VolumeManager(manager.Manager): for volume in volumes: self.driver.ensure_export(ctxt, volume) - @defer.inlineCallbacks def create_volume(self, context, volume_id): """Creates and exports the volume.""" context = context.elevated() @@ -102,19 +100,18 @@ class VolumeManager(manager.Manager): logging.debug("volume %s: creating lv of size %sG", volume_ref['name'], volume_ref['size']) - yield self.driver.create_volume(volume_ref) + self.driver.create_volume(volume_ref) logging.debug("volume %s: creating export", volume_ref['name']) - yield self.driver.create_export(context, volume_ref) + self.driver.create_export(context, volume_ref) now = datetime.datetime.utcnow() self.db.volume_update(context, volume_ref['id'], {'status': 'available', 'launched_at': now}) logging.debug("volume %s: created successfully", volume_ref['name']) - defer.returnValue(volume_id) + return volume_id - @defer.inlineCallbacks def delete_volume(self, context, volume_id): """Deletes and unexports volume.""" context = context.elevated() @@ -124,14 +121,13 @@ class VolumeManager(manager.Manager): if volume_ref['host'] != self.host: raise exception.Error("Volume is not local to this node") logging.debug("volume %s: removing export", volume_ref['name']) - yield self.driver.remove_export(context, volume_ref) + self.driver.remove_export(context, volume_ref) logging.debug("volume %s: deleting", volume_ref['name']) - yield self.driver.delete_volume(volume_ref) + self.driver.delete_volume(volume_ref) self.db.volume_destroy(context, volume_id) logging.debug("volume %s: deleted successfully", volume_ref['name']) - defer.returnValue(True) + return True - @defer.inlineCallbacks def setup_compute_volume(self, context, volume_id): """Setup remote volume on compute host. @@ -139,17 +135,16 @@ class VolumeManager(manager.Manager): context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host and FLAGS.use_local_volumes: - path = yield self.driver.local_path(volume_ref) + path = self.driver.local_path(volume_ref) else: - path = yield self.driver.discover_volume(volume_ref) - defer.returnValue(path) + path = self.driver.discover_volume(volume_ref) + return path - @defer.inlineCallbacks def remove_compute_volume(self, context, volume_id): """Remove remote volume on compute host.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host and FLAGS.use_local_volumes: - defer.returnValue(True) + return True else: - yield self.driver.undiscover_volume(volume_ref) + self.driver.undiscover_volume(volume_ref) From 15f7361f5497c3d27dcafbb27d314af76069ed42 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 8 Dec 2010 17:25:57 -0800 Subject: [PATCH 085/119] remove some unused files --- nova/process.py | 209 ------------------------------- nova/tests/process_unittest.py | 132 ------------------- nova/tests/validator_unittest.py | 42 ------- nova/validate.py | 94 -------------- 4 files changed, 477 deletions(-) delete mode 100644 nova/process.py delete mode 100644 nova/tests/process_unittest.py delete mode 100644 nova/tests/validator_unittest.py delete mode 100644 nova/validate.py diff --git a/nova/process.py b/nova/process.py deleted file mode 100644 index b33df048bf..0000000000 --- a/nova/process.py +++ /dev/null @@ -1,209 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 FathomDB Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Process pool using twisted threading -""" - -import logging -import StringIO - -from twisted.internet import defer -from twisted.internet import error -from twisted.internet import protocol -from twisted.internet import reactor - -from nova import flags -from nova.exception import ProcessExecutionError - -FLAGS = flags.FLAGS -flags.DEFINE_integer('process_pool_size', 4, - 'Number of processes to use in the process pool') - - -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to -# handle stdin -class BackRelayWithInput(protocol.ProcessProtocol): - """ - Trivial protocol for communicating with a process and turning its output - into the result of a L{Deferred}. - - @ivar deferred: A L{Deferred} which will be called back with all of stdout - and all of stderr as well (as a tuple). C{terminate_on_stderr} is true - and any bytes are received over stderr, this will fire with an - L{_ProcessExecutionError} instance and the attribute will be set to - C{None}. - - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_ProcessExecutionError} which C{deferred} fires - with earlier in this case so that users can determine when the process - has actually ended, in addition to knowing when bytes have been - received via stderr. - """ - - def __init__(self, deferred, cmd, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): - self.deferred = deferred - self.cmd = cmd - self.stdout = StringIO.StringIO() - self.stderr = StringIO.StringIO() - self.started_deferred = started_deferred - self.terminate_on_stderr = terminate_on_stderr - self.check_exit_code = check_exit_code - self.process_input = process_input - self.on_process_ended = None - - def _build_execution_error(self, exit_code=None): - return ProcessExecutionError(cmd=self.cmd, - exit_code=exit_code, - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue()) - - def errReceived(self, text): - self.stderr.write(text) - if self.terminate_on_stderr and (self.deferred is not None): - self.on_process_ended = defer.Deferred() - self.deferred.errback(self._build_execution_error()) - self.deferred = None - self.transport.loseConnection() - - def outReceived(self, text): - self.stdout.write(text) - - def processEnded(self, reason): - if self.deferred is not None: - stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - exit_code = reason.value.exitCode - if self.check_exit_code and exit_code != 0: - self.deferred.errback(self._build_execution_error(exit_code)) - else: - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me. - # If the callback throws an exception, then errback will - # be called also. However, this is what the unit tests - # test for. - exec_error = self._build_execution_error(exit_code) - self.deferred.errback(exec_error) - elif self.on_process_ended is not None: - self.on_process_ended.errback(reason) - - def connectionMade(self): - if self.started_deferred: - self.started_deferred.callback(self) - if self.process_input: - self.transport.write(str(self.process_input)) - self.transport.closeStdin() - - -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, - process_input=None, started_deferred=None, - terminate_on_stderr=False): - if process_reactor is None: - process_reactor = reactor - args = args and args or () - env = env and env and {} - deferred = defer.Deferred() - cmd = executable - if args: - cmd = " ".join([cmd] + args) - logging.debug("Running cmd: %s", cmd) - process_handler = BackRelayWithInput( - deferred, - cmd, - started_deferred=started_deferred, - check_exit_code=check_exit_code, - process_input=process_input, - terminate_on_stderr=terminate_on_stderr) - # NOTE(vish): commands come in as unicode, but self.executes needs - # strings or process.spawn raises a deprecation warning - executable = str(executable) - if not args is None: - args = [str(x) for x in args] - process_reactor.spawnProcess(process_handler, executable, - (executable,) + tuple(args), env, path) - return deferred - - -class ProcessPool(object): - """ A simple process pool implementation using Twisted's Process bits. - - This is pretty basic right now, but hopefully the API will be the correct - one so that it can be optimized later. - """ - def __init__(self, size=None): - self.size = size and size or FLAGS.process_pool_size - self._pool = defer.DeferredSemaphore(self.size) - - def simple_execute(self, cmd, **kw): - """ Weak emulation of the old utils.execute() function. - - This only exists as a way to quickly move old execute methods to - this new style of code. - - NOTE(termie): This will break on args with spaces in them. - """ - parsed = cmd.split(' ') - executable, args = parsed[0], parsed[1:] - return self.execute(executable, args, **kw) - - def execute(self, *args, **kw): - deferred = self._pool.acquire() - - def _associate_process(proto): - deferred.process = proto.transport - return proto.transport - - started = defer.Deferred() - started.addCallback(_associate_process) - kw.setdefault('started_deferred', started) - - deferred.process = None - deferred.started = started - - deferred.addCallback(lambda _: get_process_output(*args, **kw)) - deferred.addBoth(self._release) - return deferred - - def _release(self, retval=None): - self._pool.release() - return retval - - -class SharedPool(object): - _instance = None - - def __init__(self): - if SharedPool._instance is None: - self.__class__._instance = ProcessPool() - - def __getattr__(self, key): - return getattr(self._instance, key) - - -def simple_execute(cmd, **kwargs): - return SharedPool().simple_execute(cmd, **kwargs) diff --git a/nova/tests/process_unittest.py b/nova/tests/process_unittest.py deleted file mode 100644 index 67245af039..0000000000 --- a/nova/tests/process_unittest.py +++ /dev/null @@ -1,132 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -from twisted.internet import defer -from twisted.internet import reactor -from xml.etree import ElementTree - -from nova import exception -from nova import flags -from nova import process -from nova import test -from nova import utils - -FLAGS = flags.FLAGS - - -class ProcessTestCase(test.TrialTestCase): - def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) - super(ProcessTestCase, self).setUp() - - def test_execute_stdout(self): - pool = process.ProcessPool(2) - d = pool.simple_execute('echo test') - - def _check(rv): - self.assertEqual(rv[0], 'test\n') - self.assertEqual(rv[1], '') - - d.addCallback(_check) - d.addErrback(self.fail) - return d - - def test_execute_stderr(self): - pool = process.ProcessPool(2) - d = pool.simple_execute('cat BAD_FILE', check_exit_code=False) - - def _check(rv): - self.assertEqual(rv[0], '') - self.assert_('No such file' in rv[1]) - - d.addCallback(_check) - d.addErrback(self.fail) - return d - - def test_execute_unexpected_stderr(self): - pool = process.ProcessPool(2) - d = pool.simple_execute('cat BAD_FILE') - d.addCallback(lambda x: self.fail('should have raised an error')) - d.addErrback(lambda failure: failure.trap(IOError)) - return d - - def test_max_processes(self): - pool = process.ProcessPool(2) - d1 = pool.simple_execute('sleep 0.01') - d2 = pool.simple_execute('sleep 0.01') - d3 = pool.simple_execute('sleep 0.005') - d4 = pool.simple_execute('sleep 0.005') - - called = [] - - def _called(rv, name): - called.append(name) - - d1.addCallback(_called, 'd1') - d2.addCallback(_called, 'd2') - d3.addCallback(_called, 'd3') - d4.addCallback(_called, 'd4') - - # Make sure that d3 and d4 had to wait on the other two and were called - # in order - # NOTE(termie): there may be a race condition in this test if for some - # reason one of the sleeps takes longer to complete - # than it should - d4.addCallback(lambda x: self.assertEqual(called[2], 'd3')) - d4.addCallback(lambda x: self.assertEqual(called[3], 'd4')) - d4.addErrback(self.fail) - return d4 - - def test_kill_long_process(self): - pool = process.ProcessPool(2) - - d1 = pool.simple_execute('sleep 1') - d2 = pool.simple_execute('sleep 0.005') - - timeout = reactor.callLater(0.1, self.fail, 'should have been killed') - - # kill d1 and wait on it to end then cancel the timeout - d2.addCallback(lambda _: d1.process.signalProcess('KILL')) - d2.addCallback(lambda _: d1) - d2.addBoth(lambda _: timeout.active() and timeout.cancel()) - d2.addErrback(self.fail) - return d2 - - def test_process_exit_is_contained(self): - pool = process.ProcessPool(2) - - d1 = pool.simple_execute('sleep 1') - d1.addCallback(lambda x: self.fail('should have errbacked')) - d1.addErrback(lambda fail: fail.trap(IOError)) - reactor.callLater(0.05, d1.process.signalProcess, 'KILL') - - return d1 - - def test_shared_pool_is_singleton(self): - pool1 = process.SharedPool() - pool2 = process.SharedPool() - self.assertEqual(id(pool1._instance), id(pool2._instance)) - - def test_shared_pool_works_as_singleton(self): - d1 = process.simple_execute('sleep 1') - d2 = process.simple_execute('sleep 0.005') - # lp609749: would have failed with - # exceptions.AssertionError: Someone released me too many times: - # too many tokens! - return d1 diff --git a/nova/tests/validator_unittest.py b/nova/tests/validator_unittest.py deleted file mode 100644 index b5f1c0667f..0000000000 --- a/nova/tests/validator_unittest.py +++ /dev/null @@ -1,42 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import unittest - -from nova import flags -from nova import test -from nova import validate - - -class ValidationTestCase(test.TrialTestCase): - def setUp(self): - super(ValidationTestCase, self).setUp() - - def tearDown(self): - super(ValidationTestCase, self).tearDown() - - def test_type_validation(self): - self.assertTrue(type_case("foo", 5, 1)) - self.assertRaises(TypeError, type_case, "bar", "5", 1) - self.assertRaises(TypeError, type_case, None, 5, 1) - - -@validate.typetest(instanceid=str, size=int, number_of_instances=int) -def type_case(instanceid, size, number_of_instances): - return True diff --git a/nova/validate.py b/nova/validate.py deleted file mode 100644 index 7ea27daa6e..0000000000 --- a/nova/validate.py +++ /dev/null @@ -1,94 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Decorators for argument validation, courtesy of -http://rmi.net/~lutz/rangetest.html""" - - -def rangetest(**argchecks): - """Validate ranges for both + defaults""" - - def onDecorator(func): - """onCall remembers func and argchecks""" - import sys - code = func.__code__ if sys.version_info[0] == 3 else func.func_code - allargs = code.co_varnames[:code.co_argcount] - funcname = func.__name__ - - def onCall(*pargs, **kargs): - # all pargs match first N args by position - # the rest must be in kargs or omitted defaults - positionals = list(allargs) - positionals = positionals[:len(pargs)] - - for (argname, (low, high)) in argchecks.items(): - # for all args to be checked - if argname in kargs: - # was passed by name - if float(kargs[argname]) < low or \ - float(kargs[argname]) > high: - errmsg = '{0} argument "{1}" not in {2}..{3}' - errmsg = errmsg.format(funcname, argname, low, high) - raise TypeError(errmsg) - - elif argname in positionals: - # was passed by position - position = positionals.index(argname) - if float(pargs[position]) < low or \ - float(pargs[position]) > high: - errmsg = '{0} argument "{1}" with value of {4} ' \ - 'not in {2}..{3}' - errmsg = errmsg.format(funcname, argname, low, high, - pargs[position]) - raise TypeError(errmsg) - else: - pass - - return func(*pargs, **kargs) # okay: run original call - return onCall - return onDecorator - - -def typetest(**argchecks): - def onDecorator(func): - import sys - code = func.__code__ if sys.version_info[0] == 3 else func.func_code - allargs = code.co_varnames[:code.co_argcount] - funcname = func.__name__ - - def onCall(*pargs, **kargs): - positionals = list(allargs)[:len(pargs)] - for (argname, typeof) in argchecks.items(): - if argname in kargs: - if not isinstance(kargs[argname], typeof): - errmsg = '{0} argument "{1}" not of type {2}' - errmsg = errmsg.format(funcname, argname, typeof) - raise TypeError(errmsg) - elif argname in positionals: - position = positionals.index(argname) - if not isinstance(pargs[position], typeof): - errmsg = '{0} argument "{1}" with value of {2} ' \ - 'not of type {3}' - errmsg = errmsg.format(funcname, argname, - pargs[position], typeof) - raise TypeError(errmsg) - else: - pass - return func(*pargs, **kargs) - return onCall - return onDecorator From 783f4fa44b835ef6c399e18679774a2e4bc4124a Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 9 Dec 2010 10:40:07 +0000 Subject: [PATCH 086/119] fixed how the XenAPI library is loaded --- nova/virt/xenapi/vm_utils.py | 9 +++++++++ nova/virt/xenapi/vmops.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 2a75f9dbf9..2b84601f25 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -50,6 +50,15 @@ class VMHelper(): The class that wraps the helper methods together. """ def __init__(self): + return + + @classmethod + def late_import(cls): + """ + Load the XenAPI module in for helper class, if required. + This is to avoid to install the XenAPI library when other + hypervisors are used + """ global XenAPI if XenAPI is None: XenAPI = __import__('XenAPI') diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 542d4894cd..e7c3102a37 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -41,6 +41,8 @@ class VMOps(object): if XenAPI is None: XenAPI = __import__('XenAPI') self._session = session + # Load XenAPI module in the helper class + VMHelper.late_import() def list_instances(self): """ List VM instances """ From b35b86c6f415a205b6ce49164cccb2a870c46fcb Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 9 Dec 2010 13:43:54 +0000 Subject: [PATCH 087/119] fixes exception throwing with wrong instance type. --- nova/compute/instance_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/instance_types.py b/nova/compute/instance_types.py index a2679e0fc2..0e9600c00f 100644 --- a/nova/compute/instance_types.py +++ b/nova/compute/instance_types.py @@ -22,6 +22,7 @@ The built-in instance properties. """ from nova import flags +from nova.exception import ApiError FLAGS = flags.FLAGS INSTANCE_TYPES = { @@ -37,8 +38,7 @@ def get_by_type(instance_type): if instance_type is None: return FLAGS.default_instance_type if instance_type not in INSTANCE_TYPES: - raise exception.ApiError("Unknown instance type: %s", - instance_type) + raise ApiError("Unknown instance type: %s" % instance_type) return instance_type From 043d3ac3643e7183d4afe8c628ce90d62a468427 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 9 Dec 2010 11:08:24 -0600 Subject: [PATCH 088/119] Make get_diagnostics async --- nova/virt/xenapi/vmops.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index e7c3102a37..9bfd072671 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -131,13 +131,14 @@ class VMOps(object): rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) + @defer.inlineCallbacks def get_diagnostics(self, instance_id): """Return data about VM diagnostics""" - vm = VMHelper.lookup_blocking(self._session, instance_id) + vm = yield VMHelper.lookup(self._session, instance_id) if vm is None: raise Exception("instance not present %s" % instance_id) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_diagnostics(self._session, rec) + rec = yield self._session.get_xenapi().VM.get_record(vm) + defer.returnValue(VMHelper.compile_diagnostics(self._session, rec)) def get_console_output(self, instance): """ Return snapshot of console """ From 50ac7dc67686742c3e57cc3a408ca9e8c988b89b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 9 Dec 2010 17:30:03 +0000 Subject: [PATCH 089/119] filter describe volumes by supplied ids. Includes unittest. --- nova/api/ec2/cloud.py | 6 ++++-- nova/tests/cloud_unittest.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 05f8c3d0b0..ebb13aedc5 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -450,13 +450,15 @@ class CloudController(object): "Timestamp": now, "output": base64.b64encode(output)} - def describe_volumes(self, context, **kwargs): + def describe_volumes(self, context, volume_id=None, **kwargs): if context.user.is_admin(): volumes = db.volume_get_all(context) else: volumes = db.volume_get_all_by_project(context, context.project_id) - volumes = [self._format_volume(context, v) for v in volumes] + # NOTE(vish): volume_id is an optional list of volume ids to filter by. + volumes = [self._format_volume(context, v) for v in volumes + if volume_id is None or v['ec2_id'] in volume_id] return {'volumeSet': volumes} diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 9886a2449b..770c942198 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -126,6 +126,19 @@ class CloudTestCase(test.TrialTestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_volumes(self): + """Makes sure describe_volumes works and filters results.""" + vol1 = db.volume_create(self.context, {}) + vol2 = db.volume_create(self.context, {}) + result = self.cloud.describe_volumes(self.context) + self.assertEqual(len(result['volumeSet']), 2) + result = self.cloud.describe_volumes(self.context, + volume_id=[vol2['ec2_id']]) + self.assertEqual(len(result['volumeSet']), 1) + self.assertEqual(result['volumeSet'][0]['volumeId'], vol2['ec2_id']) + db.volume_destroy(self.context, vol1['id']) + db.volume_destroy(self.context, vol2['id']) + def test_console_output(self): image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type From 470dcfdf793f9a8b34c320731dcfc8d403a61bd2 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 9 Dec 2010 11:32:23 -0600 Subject: [PATCH 090/119] Import module instead of function --- nova/virt/xenapi/vm_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 2b84601f25..ff062fba01 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -23,7 +23,7 @@ import logging import urllib from twisted.internet import defer -from xml.dom.minidom import parseString +from xml.dom import minidom from nova import flags from nova import utils @@ -242,7 +242,7 @@ class VMHelper(): "Kernel": metrics["os_version"]["uname"], "Distro": metrics["os_version"]["name"]} xml = get_rrd(host_ip, record["uuid"]) - rrd = parseString(xml) + rrd = minidom.parseString(xml) for i, node in enumerate(rrd.firstChild.childNodes): # We don't want all of the extra garbage if i >= 3 and i <= 11: From 1c323efd0777587b44b275827187b7c5cd6afdc5 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 9 Dec 2010 09:57:15 -0800 Subject: [PATCH 091/119] Changed OpenStack API auth layer to inject a RequestContext rather than building one everywhere we need it. --- nova/api/openstack/__init__.py | 9 +++--- nova/api/openstack/auth.py | 4 +-- nova/api/openstack/images.py | 9 ++---- nova/api/openstack/servers.py | 41 +++++++++++---------------- nova/tests/api/openstack/fakes.py | 13 +++++++-- nova/tests/api/openstack/test_auth.py | 3 ++ 6 files changed, 37 insertions(+), 42 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index c9efe52226..b9ecbd9b8d 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -30,6 +30,7 @@ import webob.dec import webob.exc import webob +from nova import context from nova import flags from nova import utils from nova import wsgi @@ -88,9 +89,7 @@ class AuthMiddleware(wsgi.Middleware): if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - if 'nova.context' not in req.environ: - req.environ['nova.context'] = {} - req.environ['nova.context']['user'] = user + req.environ['nova.context'] = context.RequestContext(user, user) return self.application @@ -125,12 +124,12 @@ class RateLimitingMiddleware(wsgi.Middleware): If the request should be rate limited, return a 413 status with a Retry-After header giving the time when the request would succeed. """ - user_id = req.environ['nova.context']['user']['id'] action_name = self.get_action_name(req) if not action_name: # Not rate limited return self.application - delay = self.get_delay(action_name, user_id) + delay = self.get_delay(action_name, + req.environ['nova.context'].user_id) if delay: # TODO(gundlach): Get the retry-after format correct. exc = webob.exc.HTTPRequestEntityTooLarge( diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 2050359154..fcda97ab19 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -74,9 +74,7 @@ class BasicApiAuthManager(object): if delta.days >= 2: self.db.auth_destroy_token(self.context, token) else: - #TODO(gundlach): Why not just return dict(id=token.user_id)? - user = self.auth.get_user(token.user_id) - return {'id': user.id} + return self.auth.get_user(token.user_id) return None def _authorize_user(self, username, key, req): diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cdbdc9bdd8..4a0a8e6f1c 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -17,7 +17,6 @@ from webob import exc -from nova import context from nova import flags from nova import utils from nova import wsgi @@ -47,10 +46,8 @@ class Controller(wsgi.Controller): def detail(self, req): """Return all public images in detail.""" - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) try: - images = self._service.detail(ctxt) + images = self._service.detail(req.environ['nova.context']) images = nova.api.openstack.limited(images, req) except NotImplementedError: # Emulate detail() using repeated calls to show() @@ -61,9 +58,7 @@ class Controller(wsgi.Controller): def show(self, req, id): """Return data about the given image id.""" - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - return dict(image=self._service.show(ctxt, id)) + return dict(image=self._service.show(req.environ['nova.context'], id)) def delete(self, req, id): # Only public images are supported for now. diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 6f2f6fed99..7704f48f18 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -17,7 +17,6 @@ from webob import exc -from nova import context from nova import exception from nova import wsgi from nova.api.openstack import faults @@ -90,29 +89,26 @@ class Controller(wsgi.Controller): entity_maker - either _entity_detail or _entity_inst """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - instance_list = self.compute_api.get_instances(ctxt) + instance_list = self.compute_api.get_instances( + req.environ['nova.context']) limited_list = nova.api.openstack.limited(instance_list, req) res = [entity_maker(inst)['server'] for inst in limited_list] return _entity_list(res) def show(self, req, id): """ Returns server details by server id """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - inst = self.compute_api.get_instance(ctxt, int(id)) - if inst: - if inst.user_id == user_id: - return _entity_detail(inst) - raise faults.Fault(exc.HTTPNotFound()) + try: + instance = self.compute_api.get_instance( + req.environ['nova.context'], int(id)) + return _entity_detail(instance) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) def delete(self, req, id): """ Destroys a server """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) try: - self.compute_api.delete_instance(ctxt, int(id)) + self.compute_api.delete_instance(req.environ['nova.context'], + int(id)) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() @@ -123,10 +119,10 @@ class Controller(wsgi.Controller): if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - key_pair = auth_manager.AuthManager.get_key_pairs(ctxt)[0] - instances = self.compute_api.create_instances(ctxt, + key_pair = auth_manager.AuthManager.get_key_pairs( + req.environ['nova.context'])[0] + instances = self.compute_api.create_instances( + req.environ['nova.context'], instance_types.get_by_flavor_id(env['server']['flavorId']), env['server']['imageId'], display_name=env['server']['name'], @@ -137,8 +133,6 @@ class Controller(wsgi.Controller): def update(self, req, id): """ Updates the server name or password """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) inst_dict = self._deserialize(req.body, req) if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -150,7 +144,8 @@ class Controller(wsgi.Controller): update_dict['display_name'] = inst_dict['server']['name'] try: - self.compute_api.update_instance(ctxt, instance['id'], + self.compute_api.update_instance(req.environ['nova.context'], + instance['id'], **update_dict) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -159,8 +154,6 @@ class Controller(wsgi.Controller): def action(self, req, id): """ Multi-purpose method used to reboot, rebuild, and resize a server """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) input_dict = self._deserialize(req.body, req) try: reboot_type = input_dict['reboot']['type'] @@ -169,7 +162,7 @@ class Controller(wsgi.Controller): try: # TODO(gundlach): pass reboot_type, support soft reboot in # virt driver - self.compute_api.reboot(ctxt, id) + self.compute_api.reboot(req.environ['nova.context'], id) except: return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index c3f129a32d..21b8aac1c0 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -24,9 +24,10 @@ import webob import webob.dec from nova import auth -from nova import utils -from nova import flags +from nova import context from nova import exception as exc +from nova import flags +from nova import utils import nova.api.openstack.auth from nova.image import service from nova.image import glance @@ -58,7 +59,7 @@ def fake_auth_init(self): @webob.dec.wsgify def fake_wsgi(self, req): - req.environ['nova.context'] = dict(user=dict(id=1)) + req.environ['nova.context'] = context.RequestContext(1, 1) if req.body: req.environ['inst_dict'] = json.loads(req.body) return self.application @@ -171,6 +172,12 @@ class FakeToken(object): setattr(self, k, v) +class FakeRequestContext(object): + def __init__(self, user, project): + self.user_id = 1 + self.project_id = 1 + + class FakeAuthDatabase(object): data = {} diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 14e720be41..7b427c2dbb 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -26,6 +26,7 @@ import nova.api import nova.api.openstack.auth import nova.auth.manager from nova import auth +from nova import context from nova.tests.api.openstack import fakes @@ -35,6 +36,7 @@ class Test(unittest.TestCase): self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.BasicApiAuthManager, '__init__', fakes.fake_auth_init) + self.stubs.Set(context, 'RequestContext', fakes.FakeRequestContext) fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} fakes.stub_out_rate_limiting(self.stubs) @@ -131,6 +133,7 @@ class TestLimiter(unittest.TestCase): self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.BasicApiAuthManager, '__init__', fakes.fake_auth_init) + self.stubs.Set(context, 'RequestContext', fakes.FakeRequestContext) fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) From a00c8015e4ffe417f6c111a5eaf0578d9ef79b7d Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 9 Dec 2010 13:37:30 -0600 Subject: [PATCH 092/119] Added exception handling to get_rrd() --- nova/virt/xenapi/vm_utils.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index ff062fba01..77edb576e8 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -242,13 +242,14 @@ class VMHelper(): "Kernel": metrics["os_version"]["uname"], "Distro": metrics["os_version"]["name"]} xml = get_rrd(host_ip, record["uuid"]) - rrd = minidom.parseString(xml) - for i, node in enumerate(rrd.firstChild.childNodes): - # We don't want all of the extra garbage - if i >= 3 and i <= 11: - ref = node.childNodes - # Name and Value - diags[ref[0].firstChild.data] = ref[6].firstChild.data + if xml: + rrd = minidom.parseString(xml) + for i, node in enumerate(rrd.firstChild.childNodes): + # We don't want all of the extra garbage + if i >= 3 and i <= 11: + ref = node.childNodes + # Name and Value + diags[ref[0].firstChild.data] = ref[6].firstChild.data return diags except XenAPI.Failure as e: return {"Unable to retrieve diagnostics": e} @@ -256,9 +257,12 @@ class VMHelper(): def get_rrd(host, uuid): """Return the VM RRD XML as a string""" - xml = urllib.urlopen("http://%s:%s@%s/vm_rrd?uuid=%s" % ( - FLAGS.xenapi_connection_username, - FLAGS.xenapi_connection_password, - host, - uuid)) - return xml.read() + try: + xml = urllib.urlopen("http://%s:%s@%s/vm_rrd?uuid=%s" % ( + FLAGS.xenapi_connection_username, + FLAGS.xenapi_connection_password, + host, + uuid)) + return xml.read() + except IOError: + return None From 4c9bf2f01fb712a3af6a9876a175a7a0638bcd59 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 9 Dec 2010 20:18:06 +0000 Subject: [PATCH 093/119] import module and not classe directely as per Soren recommendation. --- nova/compute/instance_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/instance_types.py b/nova/compute/instance_types.py index 0e9600c00f..6e47170bde 100644 --- a/nova/compute/instance_types.py +++ b/nova/compute/instance_types.py @@ -22,7 +22,7 @@ The built-in instance properties. """ from nova import flags -from nova.exception import ApiError +from nova import exception FLAGS = flags.FLAGS INSTANCE_TYPES = { @@ -38,7 +38,7 @@ def get_by_type(instance_type): if instance_type is None: return FLAGS.default_instance_type if instance_type not in INSTANCE_TYPES: - raise ApiError("Unknown instance type: %s" % instance_type) + raise exception.ApiError("Unknown instance type: %s" % instance_type) return instance_type From 00f329d479564ad8349ed32a27990da2ed3a396e Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 14:36:23 -0800 Subject: [PATCH 094/119] make nova binaries use eventlet --- bin/nova-api | 10 +++++----- bin/nova-combined | 14 +++++++++++--- bin/nova-compute | 1 + bin/nova-network | 1 + bin/nova-scheduler | 15 ++++++--------- bin/nova-volume | 15 ++++++--------- nova/service_eventlet.py | 4 +++- 7 files changed, 33 insertions(+), 27 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a9c53dbcdf..3215ad5efa 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -32,9 +32,11 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +from nova import api from nova import flags from nova import utils from nova import server +from nova import wsgi FLAGS = flags.FLAGS flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') @@ -43,9 +45,8 @@ flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') -def main(_args): - from nova import api - from nova import wsgi +def main(): + FLAGS(sys.argv) server = wsgi.Server() server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) @@ -53,5 +54,4 @@ def main(_args): if __name__ == '__main__': - utils.default_flagfile() - server.serve('nova-api', main) + main() diff --git a/bin/nova-combined b/bin/nova-combined index 65865acd98..c865843289 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -27,8 +27,6 @@ eventlet.monkey_patch() import os import sys -from eventlet import greenthread - # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), @@ -46,6 +44,12 @@ from nova import wsgi FLAGS = flags.FLAGS flags.DEFINE_integer('api_port', 8773, 'API port') +FLAGS = flags.FLAGS +flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') +flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') +flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') +flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') + if __name__ == '__main__': FLAGS(sys.argv) @@ -57,5 +61,9 @@ if __name__ == '__main__': #objectstore = service_eventlet.Service.create(binary='nova-objectstore') service_eventlet.serve(compute, network, volume, scheduler) - wsgi.run_server(api.API(), FLAGS.api_port) + + server = wsgi.Server() + server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) + server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) + server.wait() diff --git a/bin/nova-compute b/bin/nova-compute index 600fbb8973..307f7cb591 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -39,3 +39,4 @@ from nova import service_eventlet if __name__ == '__main__': service_eventlet.serve() + service_eventlet.wait() diff --git a/bin/nova-network b/bin/nova-network index 600fbb8973..307f7cb591 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -39,3 +39,4 @@ from nova import service_eventlet if __name__ == '__main__': service_eventlet.serve() + service_eventlet.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 4d1a40cf10..e4aa0dcaeb 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -21,6 +21,9 @@ Twistd daemon for the nova scheduler nodes. """ +import eventlet +eventlet.monkey_patch() + import os import sys @@ -32,14 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service -from nova import twistd -from nova import utils - +from nova import service_eventlet if __name__ == '__main__': - utils.default_flagfile() - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() + service_eventlet.serve() + service_eventlet.wait() diff --git a/bin/nova-volume b/bin/nova-volume index e7281d6c0b..395d160418 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -21,6 +21,9 @@ Twistd daemon for the nova volume nodes. """ +import eventlet +eventlet.monkey_patch() + import os import sys @@ -32,14 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service -from nova import twistd -from nova import utils - +from nova import service_eventlet if __name__ == '__main__': - utils.default_flagfile() - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() # pylint: disable-msg=C0103 + service_eventlet.serve() + service_eventlet.wait() diff --git a/nova/service_eventlet.py b/nova/service_eventlet.py index eac45a9810..82291ffe13 100644 --- a/nova/service_eventlet.py +++ b/nova/service_eventlet.py @@ -285,4 +285,6 @@ def serve(*services): #while True: # greenthread.sleep(5) - +def wait(): + while True: + greenthread.sleep(5) From 2cad5375dc3784397ac8e6a0aa170a161e97ea7e Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 14:37:43 -0800 Subject: [PATCH 095/119] whitespace fix --- nova/service_eventlet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/service_eventlet.py b/nova/service_eventlet.py index 82291ffe13..83f0692ce1 100644 --- a/nova/service_eventlet.py +++ b/nova/service_eventlet.py @@ -285,6 +285,7 @@ def serve(*services): #while True: # greenthread.sleep(5) + def wait(): while True: greenthread.sleep(5) From 3b376b8ad167e91119e21180bbff41eceef22e26 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 15:19:56 -0800 Subject: [PATCH 096/119] get service unittests runnning again --- nova/service_eventlet.py | 20 ++++-- nova/test.py | 118 ++++++++++++++++++++++++------- nova/tests/scheduler_unittest.py | 22 +++--- nova/tests/service_unittest.py | 44 ++++++------ nova/utils.py | 7 +- run_tests.py | 4 +- 6 files changed, 149 insertions(+), 66 deletions(-) diff --git a/nova/service_eventlet.py b/nova/service_eventlet.py index 83f0692ce1..576cf8b85a 100644 --- a/nova/service_eventlet.py +++ b/nova/service_eventlet.py @@ -68,6 +68,7 @@ class Service(object): self.periodic_interval = periodic_interval super(Service, self).__init__(*args, **kwargs) self.saved_args, self.saved_kwargs = args, kwargs + self.timers = [] def start(self): manager_class = utils.import_class(self.manager_class_name) @@ -96,15 +97,17 @@ class Service(object): topic='%s.%s' % (self.topic, self.host), proxy=self) - consumer_all.attach_to_eventlet() - consumer_node.attach_to_eventlet() + self.timers.append(consumer_all.attach_to_eventlet()) + self.timers.append(consumer_node.attach_to_eventlet()) pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) + self.timers.append(pulse) if self.periodic_interval: - pulse = utils.LoopingCall(self.periodic_tasks) - pulse.start(interval=self.periodic_interval, now=False) + periodic = utils.LoopingCall(self.periodic_tasks) + periodic.start(interval=self.periodic_interval, now=False) + self.timers.append(periodic) def _create_service_ref(self, context): service_ref = db.service_create(context, @@ -156,11 +159,20 @@ class Service(object): def kill(self): """Destroy the service object in the datastore""" + self.stop() try: db.service_destroy(context.get_admin_context(), self.service_id) except exception.NotFound: logging.warn("Service killed that has no database entry") + def stop(self): + for x in self.timers: + try: + x.stop() + except Exception: + pass + self.timers = [] + def periodic_tasks(self): """Tasks to be run at a periodic interval""" self.manager.periodic_tasks(context.get_admin_context()) diff --git a/nova/test.py b/nova/test.py index bbf063aca3..e27d291662 100644 --- a/nova/test.py +++ b/nova/test.py @@ -30,6 +30,7 @@ import unittest import mox import stubout from twisted.internet import defer +from twisted.trial import unittest as trial_unittest from nova import context from nova import db @@ -54,7 +55,6 @@ def skip_if_fake(func): return func(*args, **kw) return _skipper - class TrialTestCase(unittest.TestCase): """Test case base class for all unit tests""" def setUp(self): @@ -125,29 +125,6 @@ class TrialTestCase(unittest.TestCase): for k, v in self._original_flags.iteritems(): setattr(FLAGS, k, v) - #def run(self, result=None): - # test_method = getattr(self, self._testMethodName) - # setattr(self, - # self._testMethodName, - # self._maybeInlineCallbacks(test_method, result)) - # rv = super(TrialTestCase, self).run(result) - # setattr(self, self._testMethodName, test_method) - # return rv - - #def _maybeInlineCallbacks(self, func, result): - # def _wrapped(): - # g = func() - # if isinstance(g, defer.Deferred): - # return g - # if not hasattr(g, 'send'): - # return defer.succeed(g) - - # inlined = defer.inlineCallbacks(func) - # d = inlined() - # return d - # _wrapped.func_name = func.func_name - # return _wrapped - def _monkey_patch_attach(self): self.originalAttach = rpc.Consumer.attach_to_eventlet @@ -158,3 +135,96 @@ class TrialTestCase(unittest.TestCase): _wrapped.func_name = self.originalAttach.func_name rpc.Consumer.attach_to_eventlet = _wrapped + + +class OLDTrialTestCase(trial_unittest.TestCase): + """Test case base class for all unit tests""" + def setUp(self): + """Run before each test method to initialize test environment""" + super(TrialTestCase, self).setUp() + # NOTE(vish): We need a better method for creating fixtures for tests + # now that we have some required db setup for the system + # to work properly. + self.start = datetime.datetime.utcnow() + ctxt = context.get_admin_context() + if db.network_count(ctxt) != 5: + network_manager.VlanManager().create_networks(ctxt, + FLAGS.fixed_range, + 5, 16, + FLAGS.vlan_start, + FLAGS.vpn_start) + + # emulate some of the mox stuff, we can't use the metaclass + # because it screws with our generators + self.mox = mox.Mox() + self.stubs = stubout.StubOutForTesting() + self.flag_overrides = {} + self.injected = [] + self._original_flags = FLAGS.FlagValuesDict() + + def tearDown(self): + """Runs after each test method to finalize/tear down test + environment.""" + try: + self.mox.UnsetStubs() + self.stubs.UnsetAll() + self.stubs.SmartUnsetAll() + self.mox.VerifyAll() + # NOTE(vish): Clean up any ips associated during the test. + ctxt = context.get_admin_context() + db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, + self.start) + db.network_disassociate_all(ctxt) + rpc.Consumer.attach_to_eventlet = self.originalAttach + for x in self.injected: + try: + x.stop() + except AssertionError: + pass + + if FLAGS.fake_rabbit: + fakerabbit.reset_all() + + db.security_group_destroy_all(ctxt) + super(TrialTestCase, self).tearDown() + finally: + self.reset_flags() + + def flags(self, **kw): + """Override flag variables for a test""" + for k, v in kw.iteritems(): + if k in self.flag_overrides: + self.reset_flags() + raise Exception( + 'trying to override already overriden flag: %s' % k) + self.flag_overrides[k] = getattr(FLAGS, k) + setattr(FLAGS, k, v) + + def reset_flags(self): + """Resets all flag variables for the test. Runs after each test""" + FLAGS.Reset() + for k, v in self._original_flags.iteritems(): + setattr(FLAGS, k, v) + + def run(self, result=None): + test_method = getattr(self, self._testMethodName) + setattr(self, + self._testMethodName, + self._maybeInlineCallbacks(test_method, result)) + rv = super(TrialTestCase, self).run(result) + setattr(self, self._testMethodName, test_method) + return rv + + def _maybeInlineCallbacks(self, func, result): + def _wrapped(): + g = func() + if isinstance(g, defer.Deferred): + return g + if not hasattr(g, 'send'): + return defer.succeed(g) + + inlined = defer.inlineCallbacks(func) + d = inlined() + return d + _wrapped.func_name = func.func_name + return _wrapped diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index cb5fe6b9ca..37d15567fa 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -22,7 +22,7 @@ Tests For Scheduler from nova import context from nova import db from nova import flags -from nova import service +from nova import service_eventlet as service from nova import test from nova import rpc from nova import utils @@ -122,12 +122,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-compute', 'compute', FLAGS.compute_manager) - compute1.startService() + compute1.start() compute2 = service.Service('host2', 'nova-compute', 'compute', FLAGS.compute_manager) - compute2.startService() + compute2.start() hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(len(hosts), 2) compute1.kill() @@ -139,12 +139,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-compute', 'compute', FLAGS.compute_manager) - compute1.startService() + compute1.start() compute2 = service.Service('host2', 'nova-compute', 'compute', FLAGS.compute_manager) - compute2.startService() + compute2.start() instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance() @@ -162,12 +162,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-compute', 'compute', FLAGS.compute_manager) - compute1.startService() + compute1.start() compute2 = service.Service('host2', 'nova-compute', 'compute', FLAGS.compute_manager) - compute2.startService() + compute2.start() instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): @@ -195,12 +195,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-volume', 'volume', FLAGS.volume_manager) - volume1.startService() + volume1.start() volume2 = service.Service('host2', 'nova-volume', 'volume', FLAGS.volume_manager) - volume2.startService() + volume2.start() volume_id1 = self._create_volume() volume1.create_volume(self.context, volume_id1) volume_id2 = self._create_volume() @@ -218,12 +218,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-volume', 'volume', FLAGS.volume_manager) - volume1.startService() + volume1.start() volume2 = service.Service('host2', 'nova-volume', 'volume', FLAGS.volume_manager) - volume2.startService() + volume2.start() volume_ids1 = [] volume_ids2 = [] for index in xrange(FLAGS.max_gigabytes): diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 4f8d2d550f..a2bac9af02 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -22,14 +22,11 @@ Unit Tests for remote procedure calls using queue import mox -from twisted.application.app import startApplication -from twisted.internet import defer - from nova import exception from nova import flags from nova import rpc from nova import test -from nova import service +from nova import service_eventlet as service from nova import manager FLAGS = flags.FLAGS @@ -63,7 +60,7 @@ class ServiceManagerTestCase(test.TrialTestCase): 'test', 'test', 'nova.tests.service_unittest.FakeManager') - serv.startService() + serv.start() self.assertEqual(serv.test_method(), 'manager') def test_override_manager_method(self): @@ -71,7 +68,7 @@ class ServiceManagerTestCase(test.TrialTestCase): 'test', 'test', 'nova.tests.service_unittest.FakeManager') - serv.startService() + serv.start() self.assertEqual(serv.test_method(), 'service') @@ -94,8 +91,8 @@ class ServiceTestCase(test.TrialTestCase): self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) - self.mox.StubOutWithMock( - service.task, 'LoopingCall', use_mock_anything=True) + #self.mox.StubOutWithMock( + # service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), topic=topic, proxy=mox.IsA(service.Service)).AndReturn( @@ -106,19 +103,19 @@ class ServiceTestCase(test.TrialTestCase): proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) - rpc.AdapterConsumer.attach_to_twisted() - rpc.AdapterConsumer.attach_to_twisted() + rpc.AdapterConsumer.attach_to_eventlet() + rpc.AdapterConsumer.attach_to_eventlet() # Stub out looping call a bit needlessly since we don't have an easy # way to cancel it (yet) when the tests finishes - service.task.LoopingCall(mox.IgnoreArg()).AndReturn( - service.task.LoopingCall) - service.task.LoopingCall.start(interval=mox.IgnoreArg(), - now=mox.IgnoreArg()) - service.task.LoopingCall(mox.IgnoreArg()).AndReturn( - service.task.LoopingCall) - service.task.LoopingCall.start(interval=mox.IgnoreArg(), - now=mox.IgnoreArg()) + #service.task.LoopingCall(mox.IgnoreArg()).AndReturn( + # service.task.LoopingCall) + #service.task.LoopingCall.start(interval=mox.IgnoreArg(), + # now=mox.IgnoreArg()) + #service.task.LoopingCall(mox.IgnoreArg()).AndReturn( + # service.task.LoopingCall) + #service.task.LoopingCall.start(interval=mox.IgnoreArg(), + # now=mox.IgnoreArg()) service_create = {'host': host, 'binary': binary, @@ -135,8 +132,9 @@ class ServiceTestCase(test.TrialTestCase): service.db.service_create(mox.IgnoreArg(), service_create).AndReturn(service_ref) self.mox.ReplayAll() - - startApplication(app, False) + + app.start() + app.stop() self.assert_(app) # We're testing sort of weird behavior in how report_state decides @@ -172,7 +170,7 @@ class ServiceTestCase(test.TrialTestCase): binary, topic, 'nova.tests.service_unittest.FakeManager') - serv.startService() + serv.start() serv.report_state() def test_report_state_newly_disconnected(self): @@ -202,7 +200,7 @@ class ServiceTestCase(test.TrialTestCase): binary, topic, 'nova.tests.service_unittest.FakeManager') - serv.startService() + serv.start() serv.report_state() self.assert_(serv.model_disconnected) @@ -235,7 +233,7 @@ class ServiceTestCase(test.TrialTestCase): binary, topic, 'nova.tests.service_unittest.FakeManager') - serv.startService() + serv.start() serv.model_disconnected = True serv.report_state() diff --git a/nova/utils.py b/nova/utils.py index 2c43203d88..ddbe0479e7 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -253,13 +253,18 @@ class LoopingCall(object): return done.send(True) + + self.done = done greenthread.spawn(_inner) - return done + return self.done def stop(self): self._running = False + def wait(self): + return self.done.wait() + def xhtml_escape(value): """Escapes a string so it is valid within XML or XHTML. diff --git a/run_tests.py b/run_tests.py index 883d2b768c..33bf51b18e 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,7 +48,6 @@ import sys from twisted.scripts import trial as trial_script -import unittest from nova import flags from nova import twistd @@ -62,11 +61,10 @@ from nova.tests.flags_unittest import * from nova.tests.misc_unittest import * from nova.tests.network_unittest import * #from nova.tests.objectstore_unittest import * -#from nova.tests.process_unittest import * from nova.tests.quota_unittest import * from nova.tests.rpc_unittest import * from nova.tests.scheduler_unittest import * -#from nova.tests.service_unittest import * +from nova.tests.service_unittest import * from nova.tests.twistd_unittest import * from nova.tests.validator_unittest import * from nova.tests.virt_unittest import * From b3f5aba0c465b263c1d0a15c7d249dafb3a98e6c Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 15:25:14 -0800 Subject: [PATCH 097/119] remove service and rename service_eventlet to service --- bin/nova-combined | 14 +- bin/nova-compute | 6 +- bin/nova-network | 6 +- bin/nova-scheduler | 6 +- bin/nova-volume | 6 +- nova/service.py | 169 ++++++++++++++--- nova/service_eventlet.py | 303 ------------------------------- nova/tests/scheduler_unittest.py | 2 +- nova/tests/service_unittest.py | 2 +- 9 files changed, 160 insertions(+), 354 deletions(-) delete mode 100644 nova/service_eventlet.py diff --git a/bin/nova-combined b/bin/nova-combined index c865843289..c70d1d9ef7 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -37,7 +37,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import api from nova import flags -from nova import service_eventlet +from nova import service from nova import wsgi @@ -54,13 +54,13 @@ flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') if __name__ == '__main__': FLAGS(sys.argv) - compute = service_eventlet.Service.create(binary='nova-compute') - network = service_eventlet.Service.create(binary='nova-network') - volume = service_eventlet.Service.create(binary='nova-volume') - scheduler = service_eventlet.Service.create(binary='nova-scheduler') - #objectstore = service_eventlet.Service.create(binary='nova-objectstore') + compute = service.Service.create(binary='nova-compute') + network = service.Service.create(binary='nova-network') + volume = service.Service.create(binary='nova-volume') + scheduler = service.Service.create(binary='nova-scheduler') + #objectstore = service.Service.create(binary='nova-objectstore') - service_eventlet.serve(compute, network, volume, scheduler) + service.serve(compute, network, volume, scheduler) server = wsgi.Server() server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) diff --git a/bin/nova-compute b/bin/nova-compute index 307f7cb591..4baf47e296 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -35,8 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service_eventlet +from nova import service if __name__ == '__main__': - service_eventlet.serve() - service_eventlet.wait() + service.serve() + service.wait() diff --git a/bin/nova-network b/bin/nova-network index 307f7cb591..4baf47e296 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -35,8 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service_eventlet +from nova import service if __name__ == '__main__': - service_eventlet.serve() - service_eventlet.wait() + service.serve() + service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index e4aa0dcaeb..a1b7ddf60b 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -35,8 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service_eventlet +from nova import service if __name__ == '__main__': - service_eventlet.serve() - service_eventlet.wait() + service.serve() + service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index 395d160418..ba4a3a5023 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -35,8 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -from nova import service_eventlet +from nova import service if __name__ == '__main__': - service_eventlet.serve() - service_eventlet.wait() + service.serve() + service.wait() diff --git a/nova/service.py b/nova/service.py index 55a0bb2120..576cf8b85a 100644 --- a/nova/service.py +++ b/nova/service.py @@ -17,21 +17,17 @@ # under the License. """ -A service is a very thin wrapper around a Manager object. It exposes the -manager's public methods to other components of the system via rpc. It will -report state periodically to the database and is responsible for initiating -any periodic tasts that need to be executed on a given host. - -This module contains Service, a generic baseclass for all workers. +Generic Node baseclass for all workers that run on hosts """ import inspect import logging import os +import sys -from twisted.internet import defer -from twisted.internet import task -from twisted.application import service +from eventlet import event +from eventlet import greenthread +from eventlet import greenpool from nova import context from nova import db @@ -50,8 +46,16 @@ flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) +flags.DEFINE_string('pidfile', None, + 'pidfile to use for this service') -class Service(object, service.Service): + +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + + +class Service(object): """Base class for workers that run on hosts.""" def __init__(self, host, binary, topic, manager, report_interval=None, @@ -64,8 +68,9 @@ class Service(object, service.Service): self.periodic_interval = periodic_interval super(Service, self).__init__(*args, **kwargs) self.saved_args, self.saved_kwargs = args, kwargs + self.timers = [] - def startService(self): # pylint: disable-msg C0103 + def start(self): manager_class = utils.import_class(self.manager_class_name) self.manager = manager_class(host=self.host, *self.saved_args, **self.saved_kwargs) @@ -80,26 +85,29 @@ class Service(object, service.Service): except exception.NotFound: self._create_service_ref(ctxt) - conn = rpc.Connection.instance() + conn1 = rpc.Connection.instance(new=True) + conn2 = rpc.Connection.instance(new=True) if self.report_interval: consumer_all = rpc.AdapterConsumer( - connection=conn, + connection=conn1, topic=self.topic, proxy=self) consumer_node = rpc.AdapterConsumer( - connection=conn, + connection=conn2, topic='%s.%s' % (self.topic, self.host), proxy=self) - consumer_all.attach_to_twisted() - consumer_node.attach_to_twisted() - - pulse = task.LoopingCall(self.report_state) + self.timers.append(consumer_all.attach_to_eventlet()) + self.timers.append(consumer_node.attach_to_eventlet()) + + pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) + self.timers.append(pulse) if self.periodic_interval: - pulse = task.LoopingCall(self.periodic_tasks) - pulse.start(interval=self.periodic_interval, now=False) + periodic = utils.LoopingCall(self.periodic_tasks) + periodic.start(interval=self.periodic_interval, now=False) + self.timers.append(periodic) def _create_service_ref(self, context): service_ref = db.service_create(context, @@ -114,7 +122,7 @@ class Service(object, service.Service): return getattr(manager, key) @classmethod - def create(cls, + def create(cls, host=None, binary=None, topic=None, @@ -147,24 +155,28 @@ class Service(object, service.Service): service_obj = cls(host, binary, topic, manager, report_interval, periodic_interval) - # This is the parent service that twistd will be looking for when it - # parses this file, return it so that we can get it into globals. - application = service.Application(binary) - service_obj.setServiceParent(application) - return application + return service_obj def kill(self): """Destroy the service object in the datastore""" + self.stop() try: db.service_destroy(context.get_admin_context(), self.service_id) except exception.NotFound: logging.warn("Service killed that has no database entry") + def stop(self): + for x in self.timers: + try: + x.stop() + except Exception: + pass + self.timers = [] + def periodic_tasks(self): """Tasks to be run at a periodic interval""" - yield self.manager.periodic_tasks(context.get_admin_context()) + self.manager.periodic_tasks(context.get_admin_context()) - @defer.inlineCallbacks def report_state(self): """Update the state of this service in the datastore.""" ctxt = context.get_admin_context() @@ -180,7 +192,7 @@ class Service(object, service.Service): db.service_update(ctxt, self.service_id, {'report_count': service_ref['report_count'] + 1}) - + # TODO(termie): make this pattern be more elegant. if getattr(self, "model_disconnected", False): self.model_disconnected = False @@ -191,4 +203,101 @@ class Service(object, service.Service): if not getattr(self, "model_disconnected", False): self.model_disconnected = True logging.exception("model server went away") - yield + + +def stop(pidfile): + """ + Stop the daemon + """ + # Get the pid from the pidfile + try: + pf = file(pidfile, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + pid = None + + if not pid: + message = "pidfile %s does not exist. Daemon not running?\n" + sys.stderr.write(message % pidfile) + # Not an error in a restart + return + + # Try killing the daemon process + try: + while 1: + os.kill(pid, signal.SIGKILL) + time.sleep(0.1) + except OSError, err: + err = str(err) + if err.find("No such process") > 0: + if os.path.exists(pidfile): + os.remove(pidfile) + else: + print str(err) + sys.exit(1) + + +def serve(*services): + argv = FLAGS(sys.argv) + + if not services: + services = [Service.create()] + + name = '_'.join(x.binary for x in services) + logging.debug("Serving %s" % name) + + logging.getLogger('amqplib').setLevel(logging.DEBUG) + + if not FLAGS.pidfile: + FLAGS.pidfile = '%s.pid' % name + # NOTE(vish): if we're running nodaemon, redirect the log to stdout + #if FLAGS.nodaemon and not FLAGS.logfile: + # FLAGS.logfile = "-" + #if not FLAGS.logfile: + # FLAGS.logfile = '%s.log' % name + #if not FLAGS.prefix: + # FLAGS.prefix = name + #elif FLAGS.prefix.endswith('twisted'): + # FLAGS.prefix = FLAGS.prefix.replace('twisted', name) + + action = 'start' + if len(argv) > 1: + action = argv.pop() + + if action == 'stop': + stop(FLAGS.pidfile) + sys.exit() + elif action == 'restart': + stop(FLAGS.pidfile) + elif action == 'start': + pass + else: + print 'usage: %s [options] [start|stop|restart]' % argv[0] + sys.exit(1) + + #formatter = logging.Formatter( + # '(%(name)s): %(levelname)s %(message)s') + #handler = logging.StreamHandler() + #handler.setFormatter(formatter) + #logging.getLogger().addHandler(handler) + + if FLAGS.verbose: + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.getLogger().setLevel(logging.WARNING) + + logging.debug("Full set of FLAGS:") + for flag in FLAGS: + logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) + + for x in services: + x.start() + + #while True: + # greenthread.sleep(5) + + +def wait(): + while True: + greenthread.sleep(5) diff --git a/nova/service_eventlet.py b/nova/service_eventlet.py deleted file mode 100644 index 576cf8b85a..0000000000 --- a/nova/service_eventlet.py +++ /dev/null @@ -1,303 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Generic Node baseclass for all workers that run on hosts -""" - -import inspect -import logging -import os -import sys - -from eventlet import event -from eventlet import greenthread -from eventlet import greenpool - -from nova import context -from nova import db -from nova import exception -from nova import flags -from nova import rpc -from nova import utils - - -FLAGS = flags.FLAGS -flags.DEFINE_integer('report_interval', 10, - 'seconds between nodes reporting state to datastore', - lower_bound=1) - -flags.DEFINE_integer('periodic_interval', 60, - 'seconds between running periodic tasks', - lower_bound=1) - -flags.DEFINE_string('pidfile', None, - 'pidfile to use for this service') - - -flags.DEFINE_flag(flags.HelpFlag()) -flags.DEFINE_flag(flags.HelpshortFlag()) -flags.DEFINE_flag(flags.HelpXMLFlag()) - - -class Service(object): - """Base class for workers that run on hosts.""" - - def __init__(self, host, binary, topic, manager, report_interval=None, - periodic_interval=None, *args, **kwargs): - self.host = host - self.binary = binary - self.topic = topic - self.manager_class_name = manager - self.report_interval = report_interval - self.periodic_interval = periodic_interval - super(Service, self).__init__(*args, **kwargs) - self.saved_args, self.saved_kwargs = args, kwargs - self.timers = [] - - def start(self): - manager_class = utils.import_class(self.manager_class_name) - self.manager = manager_class(host=self.host, *self.saved_args, - **self.saved_kwargs) - self.manager.init_host() - self.model_disconnected = False - ctxt = context.get_admin_context() - try: - service_ref = db.service_get_by_args(ctxt, - self.host, - self.binary) - self.service_id = service_ref['id'] - except exception.NotFound: - self._create_service_ref(ctxt) - - conn1 = rpc.Connection.instance(new=True) - conn2 = rpc.Connection.instance(new=True) - if self.report_interval: - consumer_all = rpc.AdapterConsumer( - connection=conn1, - topic=self.topic, - proxy=self) - consumer_node = rpc.AdapterConsumer( - connection=conn2, - topic='%s.%s' % (self.topic, self.host), - proxy=self) - - self.timers.append(consumer_all.attach_to_eventlet()) - self.timers.append(consumer_node.attach_to_eventlet()) - - pulse = utils.LoopingCall(self.report_state) - pulse.start(interval=self.report_interval, now=False) - self.timers.append(pulse) - - if self.periodic_interval: - periodic = utils.LoopingCall(self.periodic_tasks) - periodic.start(interval=self.periodic_interval, now=False) - self.timers.append(periodic) - - def _create_service_ref(self, context): - service_ref = db.service_create(context, - {'host': self.host, - 'binary': self.binary, - 'topic': self.topic, - 'report_count': 0}) - self.service_id = service_ref['id'] - - def __getattr__(self, key): - manager = self.__dict__.get('manager', None) - return getattr(manager, key) - - @classmethod - def create(cls, - host=None, - binary=None, - topic=None, - manager=None, - report_interval=None, - periodic_interval=None): - """Instantiates class and passes back application object. - - Args: - host, defaults to FLAGS.host - binary, defaults to basename of executable - topic, defaults to bin_name - "nova-" part - manager, defaults to FLAGS._manager - report_interval, defaults to FLAGS.report_interval - periodic_interval, defaults to FLAGS.periodic_interval - """ - if not host: - host = FLAGS.host - if not binary: - binary = os.path.basename(inspect.stack()[-1][1]) - if not topic: - topic = binary.rpartition("nova-")[2] - if not manager: - manager = FLAGS.get('%s_manager' % topic, None) - if not report_interval: - report_interval = FLAGS.report_interval - if not periodic_interval: - periodic_interval = FLAGS.periodic_interval - logging.warn("Starting %s node", topic) - service_obj = cls(host, binary, topic, manager, - report_interval, periodic_interval) - - return service_obj - - def kill(self): - """Destroy the service object in the datastore""" - self.stop() - try: - db.service_destroy(context.get_admin_context(), self.service_id) - except exception.NotFound: - logging.warn("Service killed that has no database entry") - - def stop(self): - for x in self.timers: - try: - x.stop() - except Exception: - pass - self.timers = [] - - def periodic_tasks(self): - """Tasks to be run at a periodic interval""" - self.manager.periodic_tasks(context.get_admin_context()) - - def report_state(self): - """Update the state of this service in the datastore.""" - ctxt = context.get_admin_context() - try: - try: - service_ref = db.service_get(ctxt, self.service_id) - except exception.NotFound: - logging.debug("The service database object disappeared, " - "Recreating it.") - self._create_service_ref(ctxt) - service_ref = db.service_get(ctxt, self.service_id) - - db.service_update(ctxt, - self.service_id, - {'report_count': service_ref['report_count'] + 1}) - - # TODO(termie): make this pattern be more elegant. - if getattr(self, "model_disconnected", False): - self.model_disconnected = False - logging.error("Recovered model server connection!") - - # TODO(vish): this should probably only catch connection errors - except Exception: # pylint: disable-msg=W0702 - if not getattr(self, "model_disconnected", False): - self.model_disconnected = True - logging.exception("model server went away") - - -def stop(pidfile): - """ - Stop the daemon - """ - # Get the pid from the pidfile - try: - pf = file(pidfile, 'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - pid = None - - if not pid: - message = "pidfile %s does not exist. Daemon not running?\n" - sys.stderr.write(message % pidfile) - # Not an error in a restart - return - - # Try killing the daemon process - try: - while 1: - os.kill(pid, signal.SIGKILL) - time.sleep(0.1) - except OSError, err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(pidfile): - os.remove(pidfile) - else: - print str(err) - sys.exit(1) - - -def serve(*services): - argv = FLAGS(sys.argv) - - if not services: - services = [Service.create()] - - name = '_'.join(x.binary for x in services) - logging.debug("Serving %s" % name) - - logging.getLogger('amqplib').setLevel(logging.DEBUG) - - if not FLAGS.pidfile: - FLAGS.pidfile = '%s.pid' % name - # NOTE(vish): if we're running nodaemon, redirect the log to stdout - #if FLAGS.nodaemon and not FLAGS.logfile: - # FLAGS.logfile = "-" - #if not FLAGS.logfile: - # FLAGS.logfile = '%s.log' % name - #if not FLAGS.prefix: - # FLAGS.prefix = name - #elif FLAGS.prefix.endswith('twisted'): - # FLAGS.prefix = FLAGS.prefix.replace('twisted', name) - - action = 'start' - if len(argv) > 1: - action = argv.pop() - - if action == 'stop': - stop(FLAGS.pidfile) - sys.exit() - elif action == 'restart': - stop(FLAGS.pidfile) - elif action == 'start': - pass - else: - print 'usage: %s [options] [start|stop|restart]' % argv[0] - sys.exit(1) - - #formatter = logging.Formatter( - # '(%(name)s): %(levelname)s %(message)s') - #handler = logging.StreamHandler() - #handler.setFormatter(formatter) - #logging.getLogger().addHandler(handler) - - if FLAGS.verbose: - logging.getLogger().setLevel(logging.DEBUG) - else: - logging.getLogger().setLevel(logging.WARNING) - - logging.debug("Full set of FLAGS:") - for flag in FLAGS: - logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) - - for x in services: - x.start() - - #while True: - # greenthread.sleep(5) - - -def wait(): - while True: - greenthread.sleep(5) diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 37d15567fa..f442a4bc22 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -22,7 +22,7 @@ Tests For Scheduler from nova import context from nova import db from nova import flags -from nova import service_eventlet as service +from nova import service from nova import test from nova import rpc from nova import utils diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index a2bac9af02..9f6d1af7d9 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -26,7 +26,7 @@ from nova import exception from nova import flags from nova import rpc from nova import test -from nova import service_eventlet as service +from nova import service from nova import manager FLAGS = flags.FLAGS From a1640f352806ee12f6b485a8d69a65bd42b51411 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 16:05:13 -0800 Subject: [PATCH 098/119] formatting and naming cleanup --- bin/nova-api | 6 +- bin/nova-combined | 4 +- bin/nova-compute | 4 +- bin/nova-network | 4 +- bin/nova-scheduler | 4 +- bin/nova-volume | 4 +- nova/flags.py | 4 +- nova/objectstore/image.py | 2 - nova/rpc.py | 5 +- nova/server.py | 151 ----------------------------- nova/test.py | 9 +- nova/tests/access_unittest.py | 2 +- nova/tests/api/__init__.py | 3 +- nova/tests/auth_unittest.py | 11 +-- nova/tests/cloud_unittest.py | 2 +- nova/tests/compute_unittest.py | 2 +- nova/tests/flags_unittest.py | 2 +- nova/tests/misc_unittest.py | 2 +- nova/tests/network_unittest.py | 2 +- nova/tests/objectstore_unittest.py | 4 +- nova/tests/quota_unittest.py | 2 +- nova/tests/rpc_unittest.py | 2 +- nova/tests/scheduler_unittest.py | 4 +- nova/tests/service_unittest.py | 4 +- nova/tests/virt_unittest.py | 4 +- nova/tests/volume_unittest.py | 2 +- nova/utils.py | 7 -- run_tests.py | 1 - 28 files changed, 39 insertions(+), 214 deletions(-) delete mode 100644 nova/server.py diff --git a/bin/nova-api b/bin/nova-api index 3215ad5efa..3505cefe7b 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -17,9 +17,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Nova API daemon. -""" + +"""Starter script for Nova API.""" import os import sys @@ -35,7 +34,6 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import api from nova import flags from nova import utils -from nova import server from nova import wsgi FLAGS = flags.FLAGS diff --git a/bin/nova-combined b/bin/nova-combined index c70d1d9ef7..964ffd4384 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -17,9 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova compute nodes. -""" +"""Combined starter script for Nova services.""" import eventlet eventlet.monkey_patch() diff --git a/bin/nova-compute b/bin/nova-compute index 4baf47e296..f224f0690d 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -17,9 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova compute nodes. -""" +"""Starter script for Nova Compute.""" import eventlet eventlet.monkey_patch() diff --git a/bin/nova-network b/bin/nova-network index 4baf47e296..589c75e5ad 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -17,9 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova compute nodes. -""" +"""Starter script for Nova Network.""" import eventlet eventlet.monkey_patch() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index a1b7ddf60b..cb345aa3f4 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -17,9 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova scheduler nodes. -""" +""" Starter script for Nova Scheduler.""" import eventlet eventlet.monkey_patch() diff --git a/bin/nova-volume b/bin/nova-volume index ba4a3a5023..2f74f4b9ab 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -17,9 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova volume nodes. -""" +"""Starter script for Nova Volume.""" import eventlet eventlet.monkey_patch() diff --git a/nova/flags.py b/nova/flags.py index 034b136d8b..87444565ab 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -159,12 +159,11 @@ class StrWrapper(object): return str(val) raise KeyError(name) + FLAGS = FlagValues() gflags.FLAGS = FLAGS gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) -gflags.FLAGS = FLAGS - def _wrapper(func): def _wrapped(*args, **kw): @@ -187,6 +186,7 @@ DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) DEFINE_flag = _wrapper(gflags.DEFINE_flag) + HelpFlag = gflags.HelpFlag HelpshortFlag = gflags.HelpshortFlag HelpXMLFlag = gflags.HelpXMLFlag diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 2fe0b0117c..9e56e256c1 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -26,7 +26,6 @@ Requires decryption using keys in the manifest. import binascii import glob import json -import logging import os import shutil import tarfile @@ -265,7 +264,6 @@ class Image(object): if err: raise exception.Error("Failed to decrypt initialization " "vector: %s" % err) - logging.debug(iv) _out, err = utils.execute( 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' diff --git a/nova/rpc.py b/nova/rpc.py index 652b9e4aa8..6a634a4eca 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -309,7 +309,6 @@ def call(context, topic, msg): class WaitMessage(object): def __call__(self, data, message): - LOG.debug('data %s, msg %s', data, message) """Acks message and sets result.""" message.ack() if data['failure']: @@ -332,6 +331,10 @@ def call(context, topic, msg): except StopIteration: pass consumer.close() + # NOTE(termie): this is a little bit of a change from the original + # twisted-based code where returning a Failure + # instance from a deferred call is very similar to + # raising an exception if isinstance(wait_msg.result, Exception): raise wait_msg.result return wait_msg.result diff --git a/nova/server.py b/nova/server.py deleted file mode 100644 index a060d3283e..0000000000 --- a/nova/server.py +++ /dev/null @@ -1,151 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Base functionality for nova daemons - gradually being replaced with twistd.py. -""" - -import daemon -from daemon import pidlockfile -import logging -import logging.handlers -import os -import signal -import sys -import time - -from nova import flags - - -FLAGS = flags.FLAGS -flags.DEFINE_bool('daemonize', False, 'daemonize this process') -# NOTE(termie): right now I am defaulting to using syslog when we daemonize -# it may be better to do something else -shrug- -# NOTE(Devin): I think we should let each process have its own log file -# and put it in /var/logs/nova/(appname).log -# This makes debugging much easier and cuts down on sys log -# clutter. -flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') -flags.DEFINE_string('logfile', None, 'log file to output to') -flags.DEFINE_string('logdir', None, 'directory to keep log files in ' - '(will be prepended to $logfile)') -flags.DEFINE_string('pidfile', None, 'pid file to output to') -flags.DEFINE_string('working_directory', './', 'working directory...') -flags.DEFINE_integer('uid', os.getuid(), 'uid under which to run') -flags.DEFINE_integer('gid', os.getgid(), 'gid under which to run') - - -def stop(pidfile): - """ - Stop the daemon - """ - # Get the pid from the pidfile - try: - pid = int(open(pidfile, 'r').read().strip()) - except IOError: - message = "pidfile %s does not exist. Daemon not running?\n" - sys.stderr.write(message % pidfile) - return - - # Try killing the daemon process - try: - while 1: - os.kill(pid, signal.SIGTERM) - time.sleep(0.1) - except OSError, err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(pidfile): - os.remove(pidfile) - else: - print str(err) - sys.exit(1) - - -def serve(name, main): - """Controller for server""" - argv = FLAGS(sys.argv) - - if not FLAGS.pidfile: - FLAGS.pidfile = '%s.pid' % name - - logging.debug("Full set of FLAGS: \n\n\n") - for flag in FLAGS: - logging.debug("%s : %s", flag, FLAGS.get(flag, None)) - - action = 'start' - if len(argv) > 1: - action = argv.pop() - - if action == 'stop': - stop(FLAGS.pidfile) - sys.exit() - elif action == 'restart': - stop(FLAGS.pidfile) - elif action == 'start': - pass - else: - print 'usage: %s [options] [start|stop|restart]' % argv[0] - sys.exit(1) - daemonize(argv, name, main) - - -def daemonize(args, name, main): - """Does the work of daemonizing the process""" - logging.getLogger('amqplib').setLevel(logging.WARN) - files_to_keep = [] - if FLAGS.daemonize: - logger = logging.getLogger() - formatter = logging.Formatter( - name + '(%(name)s): %(levelname)s %(message)s') - if FLAGS.use_syslog and not FLAGS.logfile: - syslog = logging.handlers.SysLogHandler(address='/dev/log') - syslog.setFormatter(formatter) - logger.addHandler(syslog) - files_to_keep.append(syslog.socket) - else: - if not FLAGS.logfile: - FLAGS.logfile = '%s.log' % name - if FLAGS.logdir: - FLAGS.logfile = os.path.join(FLAGS.logdir, FLAGS.logfile) - logfile = logging.FileHandler(FLAGS.logfile) - logfile.setFormatter(formatter) - logger.addHandler(logfile) - files_to_keep.append(logfile.stream) - stdin, stdout, stderr = None, None, None - else: - stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr - - if FLAGS.verbose: - logging.getLogger().setLevel(logging.DEBUG) - else: - logging.getLogger().setLevel(logging.WARNING) - - with daemon.DaemonContext( - detach_process=FLAGS.daemonize, - working_directory=FLAGS.working_directory, - #pidfile=pidlockfile.TimeoutPIDLockFile(FLAGS.pidfile, - # acquire_timeout=1, - # threaded=False), - stdin=stdin, - stdout=stdout, - stderr=stderr, - uid=FLAGS.uid, - gid=FLAGS.gid, - files_preserve=files_to_keep): - main(args) diff --git a/nova/test.py b/nova/test.py index e27d291662..ecc97aa4d9 100644 --- a/nova/test.py +++ b/nova/test.py @@ -55,11 +55,11 @@ def skip_if_fake(func): return func(*args, **kw) return _skipper -class TrialTestCase(unittest.TestCase): +class TestCase(unittest.TestCase): """Test case base class for all unit tests""" def setUp(self): """Run before each test method to initialize test environment""" - super(TrialTestCase, self).setUp() + super(TestCase, self).setUp() # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. @@ -105,7 +105,7 @@ class TrialTestCase(unittest.TestCase): fakerabbit.reset_all() db.security_group_destroy_all(ctxt) - super(TrialTestCase, self).tearDown() + super(TestCase, self).tearDown() finally: self.reset_flags() @@ -137,7 +137,7 @@ class TrialTestCase(unittest.TestCase): rpc.Consumer.attach_to_eventlet = _wrapped -class OLDTrialTestCase(trial_unittest.TestCase): +class TrialTestCase(trial_unittest.TestCase): """Test case base class for all unit tests""" def setUp(self): """Run before each test method to initialize test environment""" @@ -175,7 +175,6 @@ class OLDTrialTestCase(trial_unittest.TestCase): db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, self.start) db.network_disassociate_all(ctxt) - rpc.Consumer.attach_to_eventlet = self.originalAttach for x in self.injected: try: x.stop() diff --git a/nova/tests/access_unittest.py b/nova/tests/access_unittest.py index 0f66c0a26b..58fdea3b5c 100644 --- a/nova/tests/access_unittest.py +++ b/nova/tests/access_unittest.py @@ -35,7 +35,7 @@ class Context(object): pass -class AccessTestCase(test.TrialTestCase): +class AccessTestCase(test.TestCase): def setUp(self): super(AccessTestCase, self).setUp() um = manager.AuthManager() diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index cdc1bbf001..9caa8c9d0d 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -78,5 +78,4 @@ class Test(unittest.TestCase): if __name__ == '__main__': - pass - #unittest.main() + unittest.main() diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 129ff223d5..4508d67216 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -16,13 +16,10 @@ # License for the specific language governing permissions and limitations # under the License. -#import logging +import logging from M2Crypto import X509 import unittest -import eventlet -logging = eventlet.import_patched('logging') - from nova import crypto from nova import flags from nova import test @@ -329,12 +326,12 @@ class AuthManagerTestCase(object): self.assertTrue(user.is_admin()) -class AuthManagerLdapTestCase(AuthManagerTestCase, test.TrialTestCase): +class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' def __init__(self, *args, **kwargs): AuthManagerTestCase.__init__(self) - test.TrialTestCase.__init__(self, *args, **kwargs) + test.TestCase.__init__(self, *args, **kwargs) import nova.auth.fakeldap as fakeldap FLAGS.redis_db = 8 if FLAGS.flush_db: @@ -346,7 +343,7 @@ class AuthManagerLdapTestCase(AuthManagerTestCase, test.TrialTestCase): self.skip = True -class AuthManagerDbTestCase(AuthManagerTestCase, test.TrialTestCase): +class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index b7b856da52..50834d9903 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -51,7 +51,7 @@ IMAGES_PATH = os.path.join(OSS_TEMPDIR, 'images') os.makedirs(IMAGES_PATH) -class CloudTestCase(test.TrialTestCase): +class CloudTestCase(test.TestCase): def setUp(self): super(CloudTestCase, self).setUp() self.flags(connection_type='fake', images_path=IMAGES_PATH) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 67cea72c95..c6353d3575 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -35,7 +35,7 @@ from nova.compute import api as compute_api FLAGS = flags.FLAGS -class ComputeTestCase(test.TrialTestCase): +class ComputeTestCase(test.TestCase): """Test case for compute""" def setUp(self): logging.getLogger().setLevel(logging.DEBUG) diff --git a/nova/tests/flags_unittest.py b/nova/tests/flags_unittest.py index b97df075de..707300fcf6 100644 --- a/nova/tests/flags_unittest.py +++ b/nova/tests/flags_unittest.py @@ -24,7 +24,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('flags_unittest', 'foo', 'for testing purposes only') -class FlagsTestCase(test.TrialTestCase): +class FlagsTestCase(test.TestCase): def setUp(self): super(FlagsTestCase, self).setUp() diff --git a/nova/tests/misc_unittest.py b/nova/tests/misc_unittest.py index 667c63ad09..3eab1da0a1 100644 --- a/nova/tests/misc_unittest.py +++ b/nova/tests/misc_unittest.py @@ -20,7 +20,7 @@ from nova import test from nova.utils import parse_mailmap, str_dict_replace -class ProjectTestCase(test.TrialTestCase): +class ProjectTestCase(test.TestCase): def test_authors_up_to_date(self): if os.path.exists('../.bzr'): contributors = set() diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 6f4705719c..bcac205855 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -33,7 +33,7 @@ from nova.auth import manager FLAGS = flags.FLAGS -class NetworkTestCase(test.TrialTestCase): +class NetworkTestCase(test.TestCase): """Test cases for network code""" def setUp(self): super(NetworkTestCase, self).setUp() diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index 0617999234..ceac17adb6 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -54,7 +54,7 @@ os.makedirs(os.path.join(OSS_TEMPDIR, 'images')) os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) -class ObjectStoreTestCase(test.TrialTestCase): +class ObjectStoreTestCase(test.TestCase): """Test objectstore API directly.""" def setUp(self): @@ -191,7 +191,7 @@ class TestSite(server.Site): protocol = TestHTTPChannel -class S3APITestCase(test.TrialTestCase): +class S3APITestCase(test.TestCase): """Test objectstore through S3 API.""" def setUp(self): diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index 1966b51f7b..8cf2a5e546 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -32,7 +32,7 @@ from nova.api.ec2 import cloud FLAGS = flags.FLAGS -class QuotaTestCase(test.TrialTestCase): +class QuotaTestCase(test.TestCase): def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(QuotaTestCase, self).setUp() diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index c2ad5cd795..4128c35b8c 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -29,7 +29,7 @@ from nova import test FLAGS = flags.FLAGS -class RpcTestCase(test.TrialTestCase): +class RpcTestCase(test.TestCase): """Test cases for rpc""" def setUp(self): super(RpcTestCase, self).setUp() diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index f442a4bc22..d1756b8fb5 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -44,7 +44,7 @@ class TestDriver(driver.Scheduler): return 'named_host' -class SchedulerTestCase(test.TrialTestCase): +class SchedulerTestCase(test.TestCase): """Test case for scheduler""" def setUp(self): super(SchedulerTestCase, self).setUp() @@ -73,7 +73,7 @@ class SchedulerTestCase(test.TrialTestCase): scheduler.named_method(ctxt, 'topic', num=7) -class SimpleDriverTestCase(test.TrialTestCase): +class SimpleDriverTestCase(test.TestCase): """Test case for simple driver""" def setUp(self): super(SimpleDriverTestCase, self).setUp() diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 9f6d1af7d9..c94af4a230 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -45,7 +45,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.TrialTestCase): +class ServiceManagerTestCase(test.TestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -72,7 +72,7 @@ class ServiceManagerTestCase(test.TrialTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.TrialTestCase): +class ServiceTestCase(test.TestCase): """Test cases for Services""" def setUp(self): diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index a4a8d3acf8..85e5698589 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -30,7 +30,7 @@ FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') -class LibvirtConnTestCase(test.TrialTestCase): +class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() self.manager = manager.AuthManager() @@ -123,7 +123,7 @@ class LibvirtConnTestCase(test.TrialTestCase): self.manager.delete_user(self.user) -class NWFilterTestCase(test.TrialTestCase): +class NWFilterTestCase(test.TestCase): def setUp(self): super(NWFilterTestCase, self).setUp() diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 93d2ceab79..b13455fb07 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -31,7 +31,7 @@ from nova import utils FLAGS = flags.FLAGS -class VolumeTestCase(test.TrialTestCase): +class VolumeTestCase(test.TestCase): """Test Case for volumes.""" def setUp(self): diff --git a/nova/utils.py b/nova/utils.py index ddbe0479e7..22bf5d8cf4 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -129,13 +129,6 @@ def debug(arg): def runthis(prompt, cmd, check_exit_code=True): logging.debug("Running %s" % (cmd)) rv, err = execute(cmd, check_exit_code=check_exit_code) - #exit_code = subprocess.call(cmd.split(" ")) - #logging.debug(prompt % (exit_code)) - #if check_exit_code and exit_code != 0: - # raise ProcessExecutionError(exit_code=exit_code, - # stdout=None, - # stderr=None, - # cmd=cmd) def generate_uid(topic, size=8): diff --git a/run_tests.py b/run_tests.py index 33bf51b18e..6d7830a297 100644 --- a/run_tests.py +++ b/run_tests.py @@ -95,7 +95,6 @@ if __name__ == '__main__': else: from nova.tests.real_flags import * - # Establish redirect for STDERR sys.stderr.flush() err = open(FLAGS.tests_stderr, 'w+', 0) From af5c175dbc77048fb74311bf92569866676eee9c Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 16:18:52 -0800 Subject: [PATCH 099/119] removed a few more references to twisted --- nova/rpc.py | 6 ++- nova/service.py | 2 - nova/virt/images.py | 2 +- nova/virt/xenapi/network_utils.py | 9 ++--- nova/virt/xenapi/vmops.py | 61 ++++++++++++++----------------- 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 6a634a4eca..b5df4904b1 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -237,7 +237,9 @@ class DirectPublisher(Publisher): def msg_reply(msg_id, reply=None, failure=None): """Sends a reply or an error on the channel signified by msg_id - failure should be a twisted failure object""" + failure should be a sys.exc_info() tuple. + + """ if failure: message = str(failure[1]) tb = traceback.format_exception(*failure) @@ -332,7 +334,7 @@ def call(context, topic, msg): pass consumer.close() # NOTE(termie): this is a little bit of a change from the original - # twisted-based code where returning a Failure + # non-eventlet code where returning a Failure # instance from a deferred call is very similar to # raising an exception if isinstance(wait_msg.result, Exception): diff --git a/nova/service.py b/nova/service.py index 576cf8b85a..5c171b2ae8 100644 --- a/nova/service.py +++ b/nova/service.py @@ -258,8 +258,6 @@ def serve(*services): # FLAGS.logfile = '%s.log' % name #if not FLAGS.prefix: # FLAGS.prefix = name - #elif FLAGS.prefix.endswith('twisted'): - # FLAGS.prefix = FLAGS.prefix.replace('twisted', name) action = 'start' if len(argv) > 1: diff --git a/nova/virt/images.py b/nova/virt/images.py index 4d7c65f126..1c9b2e0934 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -50,7 +50,7 @@ def _fetch_s3_image(image, path, user, project): # This should probably move somewhere else, like e.g. a download_as # method on User objects and at the same time get rewritten to use - # twisted web client. + # a web client. headers = {} headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xenapi/network_utils.py index 8cb4cce3a7..d8632f3939 100644 --- a/nova/virt/xenapi/network_utils.py +++ b/nova/virt/xenapi/network_utils.py @@ -20,8 +20,6 @@ records and their attributes like bridges, PIFs, QoS, as well as their lookup functions. """ -from twisted.internet import defer - class NetworkHelper(): """ @@ -31,14 +29,13 @@ class NetworkHelper(): return @classmethod - @defer.inlineCallbacks def find_network_with_bridge(cls, session, bridge): """ Return the network on which the bridge is attached, if found """ expr = 'field "bridge" = "%s"' % bridge - networks = yield session.call_xenapi('network.get_all_records_where', - expr) + networks = session.call_xenapi('network.get_all_records_where', + expr) if len(networks) == 1: - defer.returnValue(networks.keys()[0]) + return networks.keys()[0] elif len(networks) > 1: raise Exception('Found non-unique network for bridge %s' % bridge) else: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d36cdaea5b..0223e512a6 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -20,8 +20,6 @@ Management class for VM-related functions (spawn, reboot, etc). import logging -from twisted.internet import defer - from nova import db from nova import context from nova.auth.manager import AuthManager @@ -46,10 +44,9 @@ class VMOps(object): return [self._session.get_xenapi().VM.get_name_label(vm) \ for vm in self._session.get_xenapi().VM.get_all()] - @defer.inlineCallbacks def spawn(self, instance): """ Create VM instance """ - vm = yield VMHelper.lookup(self._session, instance.name) + vm = VMHelper.lookup(self._session, instance.name) if vm is not None: raise Exception('Attempted to create non-unique name %s' % instance.name) @@ -57,66 +54,64 @@ class VMOps(object): bridge = db.project_get_network(context.get_admin_context(), instance.project_id).bridge network_ref = \ - yield NetworkHelper.find_network_with_bridge(self._session, bridge) + NetworkHelper.find_network_with_bridge(self._session, bridge) user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - vdi_uuid = yield VMHelper.fetch_image(self._session, - instance.image_id, user, project, True) - kernel = yield VMHelper.fetch_image(self._session, - instance.kernel_id, user, project, False) - ramdisk = yield VMHelper.fetch_image(self._session, - instance.ramdisk_id, user, project, False) - vdi_ref = yield self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - vm_ref = yield VMHelper.create_vm(self._session, - instance, kernel, ramdisk) - yield VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) + vdi_uuid = VMHelper.fetch_image( + self._session, instance.image_id, user, project, True) + kernel = VMHelper.fetch_image( + self._session, instance.kernel_id, user, project, False) + ramdisk = VMHelper.fetch_image( + self._session, instance.ramdisk_id, user, project, False) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) + vm_ref = VMHelper.create_vm( + self._session, instance, kernel, ramdisk) + VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) if network_ref: - yield VMHelper.create_vif(self._session, vm_ref, - network_ref, instance.mac_address) + VMHelper.create_vif(self._session, vm_ref, + network_ref, instance.mac_address) logging.debug('Starting VM %s...', vm_ref) - yield self._session.call_xenapi('VM.start', vm_ref, False, False) + self._session.call_xenapi('VM.start', vm_ref, False, False) logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - @defer.inlineCallbacks def reboot(self, instance): """ Reboot VM instance """ instance_name = instance.name - vm = yield VMHelper.lookup(self._session, instance_name) + vm = VMHelper.lookup(self._session, instance_name) if vm is None: raise Exception('instance not present %s' % instance_name) - task = yield self._session.call_xenapi('Async.VM.clean_reboot', vm) - yield self._session.wait_for_task(task) + task = self._session.call_xenapi('Async.VM.clean_reboot', vm) + self._session.wait_for_task(task) - @defer.inlineCallbacks def destroy(self, instance): """ Destroy VM instance """ - vm = yield VMHelper.lookup(self._session, instance.name) + vm = VMHelper.lookup(self._session, instance.name) if vm is None: # Don't complain, just return. This lets us clean up instances # that have already disappeared from the underlying platform. - defer.returnValue(None) + return # Get the VDIs related to the VM - vdis = yield VMHelper.lookup_vm_vdis(self._session, vm) + vdis = VMHelper.lookup_vm_vdis(self._session, vm) try: - task = yield self._session.call_xenapi('Async.VM.hard_shutdown', + task = self._session.call_xenapi('Async.VM.hard_shutdown', vm) - yield self._session.wait_for_task(task) + self._session.wait_for_task(task) except XenAPI.Failure, exc: logging.warn(exc) # Disk clean-up if vdis: for vdi in vdis: try: - task = yield self._session.call_xenapi('Async.VDI.destroy', - vdi) - yield self._session.wait_for_task(task) + task = self._session.call_xenapi('Async.VDI.destroy', + vdi) + self._session.wait_for_task(task) except XenAPI.Failure, exc: logging.warn(exc) try: - task = yield self._session.call_xenapi('Async.VM.destroy', vm) - yield self._session.wait_for_task(task) + task = self._session.call_xenapi('Async.VM.destroy', vm) + self._session.wait_for_task(task) except XenAPI.Failure, exc: logging.warn(exc) From 4d06429290a373ae3a42b1f9b58d7253d269e048 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 9 Dec 2010 16:45:51 -0800 Subject: [PATCH 100/119] add back utils.default_flagflie --- bin/nova-api | 1 + bin/nova-combined | 2 ++ bin/nova-compute | 2 ++ bin/nova-network | 2 ++ bin/nova-scheduler | 2 ++ bin/nova-volume | 2 ++ 6 files changed, 11 insertions(+) diff --git a/bin/nova-api b/bin/nova-api index 3505cefe7b..210486666f 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -52,4 +52,5 @@ def main(): if __name__ == '__main__': + utils.default_flagfile() main() diff --git a/bin/nova-combined b/bin/nova-combined index 964ffd4384..e82b9c9b6f 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -36,6 +36,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import api from nova import flags from nova import service +from nova import utils from nova import wsgi @@ -50,6 +51,7 @@ flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') if __name__ == '__main__': + utils.default_flagfile() FLAGS(sys.argv) compute = service.Service.create(binary='nova-compute') diff --git a/bin/nova-compute b/bin/nova-compute index f224f0690d..8b02cd0f54 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -34,7 +34,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) from nova import service +from nova import utils if __name__ == '__main__': + utils.default_flagfile() service.serve() service.wait() diff --git a/bin/nova-network b/bin/nova-network index 589c75e5ad..01cc9d00db 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -34,7 +34,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) from nova import service +from nova import utils if __name__ == '__main__': + utils.default_flagfile() service.serve() service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index cb345aa3f4..a46dd8dda2 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -34,7 +34,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) from nova import service +from nova import utils if __name__ == '__main__': + utils.default_flagfile() service.serve() service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index 2f74f4b9ab..efb837ba21 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -34,7 +34,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) from nova import service +from nova import utils if __name__ == '__main__': + utils.default_flagfile() service.serve() service.wait() From c835c441981a17764931390bc1ace6121ab100a4 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 10 Dec 2010 11:53:17 -0800 Subject: [PATCH 101/119] port new patches --- nova/virt/xenapi/vmops.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 13871b4790..b6b92b9264 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -126,14 +126,13 @@ class VMOps(object): rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) - @defer.inlineCallbacks def get_diagnostics(self, instance_id): """Return data about VM diagnostics""" - vm = yield VMHelper.lookup(self._session, instance_id) + vm = VMHelper.lookup(self._session, instance_id) if vm is None: raise Exception("instance not present %s" % instance_id) - rec = yield self._session.get_xenapi().VM.get_record(vm) - defer.returnValue(VMHelper.compile_diagnostics(self._session, rec)) + rec = self._session.get_xenapi().VM.get_record(vm) + return VMHelper.compile_diagnostics(self._session, rec) def get_console_output(self, instance): """ Return snapshot of console """ From 68dbbbba34af234f2770b40c03e4e4bfa5ad78d8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Dec 2010 21:09:37 +0000 Subject: [PATCH 102/119] Includes kernel and ramdisk on register. Additinally removes a couple lines of cruft --- nova/objectstore/image.py | 48 ++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 7292dbab8f..ed0f75cd27 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -21,15 +21,12 @@ Take uploaded bucket contents and register them as disk images (AMIs). Requires decryption using keys in the manifest. """ -# TODO(jesse): Got these from Euca2ools, will need to revisit them - import binascii import glob import json import os import shutil import tarfile -import tempfile from xml.etree import ElementTree from nova import exception @@ -185,33 +182,38 @@ class Image(object): manifest = ElementTree.fromstring(bucket_object[manifest_path].read()) image_type = 'machine' - try: - kernel_id = manifest.find("machine_configuration/kernel_id").text - if kernel_id == 'true': - image_type = 'kernel' - except: - kernel_id = None - - try: - ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text - if ramdisk_id == 'true': - image_type = 'ramdisk' - except: - ramdisk_id = None - info = { 'imageId': image_id, 'imageLocation': image_location, 'imageOwnerId': context.project_id, 'isPublic': False, # FIXME: grab public from manifest - 'architecture': 'x86_64', # FIXME: grab architecture from manifest - 'imageType': image_type} + 'architecture': 'x86_64', + 'imageType': 'machine' + } - if kernel_id: - info['kernelId'] = kernel_id + manifest = ElementTree.fromstring(bucket_object[manifest_path].read()) - if ramdisk_id: - info['ramdiskId'] = ramdisk_id + try: + architecture = manifest.find("machine_configuration/kernel_id").text + info['architecture'] = architecture + except: + pass + try: + kernel_id = manifest.find("machine_configuration/kernel_id").text + if kernel_id == 'true': + info['imageType'] = 'kernel' + else: + info['kernelId'] = kernel_id + except: + pass + try: + ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text + if ramdisk_id == 'true': + info['imageType'] = 'ramdisk' + else: + info['ramdiskId'] = ramdisk_id + except: + pass def write_state(state): info['imageState'] = state From f09b008388b9ed8dbd1d3f74cb1e9f2a458a3000 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 10 Dec 2010 21:15:33 +0000 Subject: [PATCH 103/119] fix pep8 --- nova/objectstore/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index ed0f75cd27..c4d752124c 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -194,8 +194,8 @@ class Image(object): manifest = ElementTree.fromstring(bucket_object[manifest_path].read()) try: - architecture = manifest.find("machine_configuration/kernel_id").text - info['architecture'] = architecture + arch = manifest.find("machine_configuration/kernel_id").text + info['architecture'] = arch except: pass try: From d60ed93a22b5cfb7dcaad2882b28a5a37c797af9 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Fri, 10 Dec 2010 16:03:30 -0600 Subject: [PATCH 104/119] Fixed spelling errors in index.rst --- doc/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index dd01b20602..b9ba6208a4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,7 +26,7 @@ Nova is written with the following design guidelines in mind: * **Component based architecture**: Quickly add new behaviors * **Highly available**: Scale to very serious workloads -* **Fault-Tollerant**: Isloated processes avoid cascading failures +* **Fault-Tolerant**: Isolated processes avoid cascading failures * **Recoverable**: Failures should be easy to diagnose, debug, and rectify * **Open Standards**: Be a reference implementation for a community-driven api * **API Compatibility**: Nova strives to provide API-compatible with popular systems like Amazon EC2 From a6645d8a431ed933eef4ea6c42c0224ead6f2272 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Sat, 11 Dec 2010 15:10:24 -0500 Subject: [PATCH 105/119] Initial i18n commit for endpoints. All endpoints must install gettext, which injects the _ function into the builtins. --- bin/nova-api | 3 +++ bin/nova-compute | 3 +++ bin/nova-dhcpbridge | 3 +++ nova/tests/__init__.py | 5 +++++ run_tests.py | 3 +++ 5 files changed, 17 insertions(+) diff --git a/bin/nova-api b/bin/nova-api index a9c53dbcdf..2ae6a099af 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -21,6 +21,7 @@ Nova API daemon. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import flags from nova import utils from nova import server diff --git a/bin/nova-compute b/bin/nova-compute index ac6378f754..f57b68584b 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -21,6 +21,7 @@ Twistd daemon for the nova compute nodes. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service from nova import twistd from nova import utils diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 17c62da0a0..81b9b6dd3f 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -21,6 +21,7 @@ Handle lease database updates from DHCP servers. """ +import gettext import logging import os import sys @@ -33,6 +34,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import context from nova import db from nova import flags diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index aaf213923e..8dc87d0e2d 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -29,3 +29,8 @@ .. moduleauthor:: Manish Singh .. moduleauthor:: Andy Smith """ + +# See http://code.google.com/p/python-nose/issues/detail?id=373 +# The code below enables nosetests to work with i18n _() blocks +import __builtin__ +setattr(__builtin__, '_', lambda x: x) diff --git a/run_tests.py b/run_tests.py index 3d427d8af2..37a548e4ce 100644 --- a/run_tests.py +++ b/run_tests.py @@ -40,9 +40,12 @@ Due to our use of multiprocessing it we frequently get some ignorable """ import __main__ +import gettext import os import sys +gettext.install('nova', unicode=1) + from twisted.scripts import trial as trial_script from nova import flags From 8e642730b1e32477bcd124592f2c9e00857da1b9 Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Mon, 13 Dec 2010 20:02:27 +0300 Subject: [PATCH 106/119] Added Twisted version dependency into pip-requires --- tools/pip-requires | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/pip-requires b/tools/pip-requires index 17a1a4c5c1..52451b8cbd 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -21,3 +21,4 @@ mox==0.5.0 greenlet==0.3.1 nose bzr +Twisted>=10.1.0 \ No newline at end of file From c06ec98897e7c3e33f15d45ba2704b0d4b77a453 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Mon, 13 Dec 2010 18:56:07 +0000 Subject: [PATCH 107/119] Add myself. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index a1703b279d..64aabbdbc7 100644 --- a/Authors +++ b/Authors @@ -3,6 +3,7 @@ Anne Gentle Anthony Young Armando Migliaccio Chris Behrens +Chmouel Boudjnah Dean Troyer Devin Carlen Eric Day From 72b18d065669a01d8d083aa3edcc726be9be6547 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 14 Dec 2010 00:20:27 +0000 Subject: [PATCH 108/119] simplified version using original logic --- nova/objectstore/image.py | 51 ++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index c4d752124c..34a90b0a27 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -21,6 +21,7 @@ Take uploaded bucket contents and register them as disk images (AMIs). Requires decryption using keys in the manifest. """ + import binascii import glob import json @@ -182,38 +183,38 @@ class Image(object): manifest = ElementTree.fromstring(bucket_object[manifest_path].read()) image_type = 'machine' + try: + kernel_id = manifest.find("machine_configuration/kernel_id").text + if kernel_id == 'true': + image_type = 'kernel' + except: + kernel_id = None + + try: + ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text + if ramdisk_id == 'true': + image_type = 'ramdisk' + except: + ramdisk_id = None + + try: + arch = manifest.find("machine_configuration/architecture").text + except: + arch = 'x86_64' + info = { 'imageId': image_id, 'imageLocation': image_location, 'imageOwnerId': context.project_id, 'isPublic': False, # FIXME: grab public from manifest - 'architecture': 'x86_64', - 'imageType': 'machine' - } + 'architecture': arch, + 'imageType': image_type} - manifest = ElementTree.fromstring(bucket_object[manifest_path].read()) + if kernel_id: + info['kernelId'] = kernel_id - try: - arch = manifest.find("machine_configuration/kernel_id").text - info['architecture'] = arch - except: - pass - try: - kernel_id = manifest.find("machine_configuration/kernel_id").text - if kernel_id == 'true': - info['imageType'] = 'kernel' - else: - info['kernelId'] = kernel_id - except: - pass - try: - ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text - if ramdisk_id == 'true': - info['imageType'] = 'ramdisk' - else: - info['ramdiskId'] = ramdisk_id - except: - pass + if ramdisk_id: + info['ramdiskId'] = ramdisk_id def write_state(state): info['imageState'] = state From d3a41eff912762dddd1516006da197f99af53b4e Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Tue, 14 Dec 2010 14:54:50 +0300 Subject: [PATCH 109/119] Added my contacts to Authors file --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index c9bf3b67c9..2810a1ddb9 100644 --- a/Authors +++ b/Authors @@ -28,3 +28,4 @@ Trey Morris Vishvananda Ishaya Youcef Laribi Zhixue Wu +Eldar Nugaev \ No newline at end of file From 797e3f8a1cc72599aa8540b5655e29da8975e56f Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Tue, 14 Dec 2010 18:22:03 -0500 Subject: [PATCH 110/119] For some reason, I forgot to commit the other endpoints... --- bin/nova-import-canonical-imagestore | 3 +++ bin/nova-instancemonitor | 3 +++ bin/nova-manage | 3 +++ bin/nova-network | 3 +++ bin/nova-objectstore | 3 +++ bin/nova-scheduler | 3 +++ bin/nova-volume | 3 +++ 7 files changed, 21 insertions(+) diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 4ed9e8365e..036b41e48b 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -21,6 +21,7 @@ Download images from Canonical Image Store """ +import gettext import json import os import tempfile @@ -37,6 +38,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import flags from nova import utils from nova.objectstore import image diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 9b6c40e820..5dac3ffe60 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -21,6 +21,7 @@ Daemon for Nova RRD based instance resource monitoring. """ +import gettext import os import logging import sys @@ -34,6 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import utils from nova import twistd from nova.compute import monitor diff --git a/bin/nova-manage b/bin/nova-manage index 62eec8353f..0c1b621ed7 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -53,6 +53,7 @@ CLI interface for nova management. """ +import gettext import logging import os import sys @@ -68,6 +69,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import context from nova import db from nova import exception diff --git a/bin/nova-network b/bin/nova-network index d1fb552612..86d04c723d 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -21,6 +21,7 @@ Twistd daemon for the nova network nodes. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service from nova import twistd from nova import utils diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 00ae27af93..9fbe228a2c 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -21,6 +21,7 @@ Twisted daemon for nova objectstore. Supports S3 API. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import flags from nova import utils from nova import twistd diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 4d1a40cf10..41e1937c13 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -21,6 +21,7 @@ Twistd daemon for the nova scheduler nodes. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service from nova import twistd from nova import utils diff --git a/bin/nova-volume b/bin/nova-volume index e7281d6c0b..4f2e962689 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -21,6 +21,7 @@ Twistd daemon for the nova volume nodes. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service from nova import twistd from nova import utils From a2a8406b5d793545c8ecb359e18b80bba618c509 Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 14 Dec 2010 16:05:39 -0800 Subject: [PATCH 111/119] updates per review --- bin/nova-api | 9 ++-- bin/nova-combined | 3 -- bin/nova-scheduler | 2 +- nova/compute/manager.py | 4 +- nova/manager.py | 2 +- nova/rpc.py | 1 - nova/service.py | 69 +------------------------------ nova/tests/service_unittest.py | 13 ------ nova/utils.py | 2 +- nova/virt/fake.py | 4 +- nova/virt/libvirt_conn.py | 14 +++---- nova/virt/xenapi/network_utils.py | 5 +-- nova/virt/xenapi/vmops.py | 3 +- nova/virt/xenapi_conn.py | 6 ++- nova/volume/driver.py | 25 +++++------ run_tests.py | 5 --- 16 files changed, 35 insertions(+), 132 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 210486666f..3f433ea6dd 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -36,6 +36,7 @@ from nova import flags from nova import utils from nova import wsgi + FLAGS = flags.FLAGS flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') @@ -43,14 +44,10 @@ flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') -def main(): +if __name__ == '__main__': + utils.default_flagfile() FLAGS(sys.argv) server = wsgi.Server() server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) server.wait() - - -if __name__ == '__main__': - utils.default_flagfile() - main() diff --git a/bin/nova-combined b/bin/nova-combined index e82b9c9b6f..5f635b3a3d 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -40,9 +40,6 @@ from nova import utils from nova import wsgi -FLAGS = flags.FLAGS -flags.DEFINE_integer('api_port', 8773, 'API port') - FLAGS = flags.FLAGS flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') diff --git a/bin/nova-scheduler b/bin/nova-scheduler index a46dd8dda2..59cb060d43 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -17,7 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" Starter script for Nova Scheduler.""" +"""Starter script for Nova Scheduler.""" import eventlet eventlet.monkey_patch() diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 051ce579d8..f90f28b78d 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -203,7 +203,7 @@ class ComputeManager(manager.Manager): volume_id, mountpoint) instance_ref = self.db.instance_get(context, instance_id) dev_path = self.volume_manager.setup_compute_volume(context, - volume_id) + volume_id) try: self.driver.attach_volume(instance_ref['name'], dev_path, @@ -238,7 +238,7 @@ class ComputeManager(manager.Manager): instance_ref['name']) else: self.driver.detach_volume(instance_ref['name'], - volume_ref['mountpoint']) + volume_ref['mountpoint']) self.volume_manager.remove_compute_volume(context, volume_id) self.db.volume_detached(context, volume_id) return True diff --git a/nova/manager.py b/nova/manager.py index a343d7fc6a..3d38504bd8 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -68,7 +68,7 @@ class Manager(base.Base): def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval""" - return + pass def init_host(self): """Do any initialization that needs to be run if this is a standalone diff --git a/nova/rpc.py b/nova/rpc.py index b5df4904b1..6a3f552dbd 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -188,7 +188,6 @@ class AdapterConsumer(TopicConsumer): node_func = getattr(self.proxy, str(method)) node_args = dict((str(k), v) for k, v in args.iteritems()) # NOTE(vish): magic is fun! - # pylint: disable-msg=W0142 try: rval = node_func(context=ctxt, **node_args) if msg_id: diff --git a/nova/service.py b/nova/service.py index 5c171b2ae8..3d40e83a6c 100644 --- a/nova/service.py +++ b/nova/service.py @@ -205,39 +205,6 @@ class Service(object): logging.exception("model server went away") -def stop(pidfile): - """ - Stop the daemon - """ - # Get the pid from the pidfile - try: - pf = file(pidfile, 'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - pid = None - - if not pid: - message = "pidfile %s does not exist. Daemon not running?\n" - sys.stderr.write(message % pidfile) - # Not an error in a restart - return - - # Try killing the daemon process - try: - while 1: - os.kill(pid, signal.SIGKILL) - time.sleep(0.1) - except OSError, err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(pidfile): - os.remove(pidfile) - else: - print str(err) - sys.exit(1) - - def serve(*services): argv = FLAGS(sys.argv) @@ -247,38 +214,7 @@ def serve(*services): name = '_'.join(x.binary for x in services) logging.debug("Serving %s" % name) - logging.getLogger('amqplib').setLevel(logging.DEBUG) - - if not FLAGS.pidfile: - FLAGS.pidfile = '%s.pid' % name - # NOTE(vish): if we're running nodaemon, redirect the log to stdout - #if FLAGS.nodaemon and not FLAGS.logfile: - # FLAGS.logfile = "-" - #if not FLAGS.logfile: - # FLAGS.logfile = '%s.log' % name - #if not FLAGS.prefix: - # FLAGS.prefix = name - - action = 'start' - if len(argv) > 1: - action = argv.pop() - - if action == 'stop': - stop(FLAGS.pidfile) - sys.exit() - elif action == 'restart': - stop(FLAGS.pidfile) - elif action == 'start': - pass - else: - print 'usage: %s [options] [start|stop|restart]' % argv[0] - sys.exit(1) - - #formatter = logging.Formatter( - # '(%(name)s): %(levelname)s %(message)s') - #handler = logging.StreamHandler() - #handler.setFormatter(formatter) - #logging.getLogger().addHandler(handler) + logging.getLogger('amqplib').setLevel(logging.WARN) if FLAGS.verbose: logging.getLogger().setLevel(logging.DEBUG) @@ -292,9 +228,6 @@ def serve(*services): for x in services: x.start() - #while True: - # greenthread.sleep(5) - def wait(): while True: diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index c94af4a230..6bdc7071cb 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -91,8 +91,6 @@ class ServiceTestCase(test.TestCase): self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) - #self.mox.StubOutWithMock( - # service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), topic=topic, proxy=mox.IsA(service.Service)).AndReturn( @@ -106,17 +104,6 @@ class ServiceTestCase(test.TestCase): rpc.AdapterConsumer.attach_to_eventlet() rpc.AdapterConsumer.attach_to_eventlet() - # Stub out looping call a bit needlessly since we don't have an easy - # way to cancel it (yet) when the tests finishes - #service.task.LoopingCall(mox.IgnoreArg()).AndReturn( - # service.task.LoopingCall) - #service.task.LoopingCall.start(interval=mox.IgnoreArg(), - # now=mox.IgnoreArg()) - #service.task.LoopingCall(mox.IgnoreArg()).AndReturn( - # service.task.LoopingCall) - #service.task.LoopingCall.start(interval=mox.IgnoreArg(), - # now=mox.IgnoreArg()) - service_create = {'host': host, 'binary': binary, 'topic': topic, diff --git a/nova/utils.py b/nova/utils.py index 22bf5d8cf4..5f2d472021 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -241,7 +241,7 @@ class LoopingCall(object): self.f(*self.args, **self.kw) greenthread.sleep(interval) except Exception: - logging.exception('hhmm') + logging.exception('in looping call') done.send_exception(*sys.exc_info()) return diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 91dc8173b3..77bc926c20 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -122,13 +122,13 @@ class FakeConnection(object): """ Rescue the specified instance. """ - return + pass def unrescue(self, instance): """ Unrescue the specified instance. """ - return + pass def destroy(self, instance): """ diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 715e7234cf..ba51f8f698 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -215,7 +215,7 @@ class LibvirtConnection(object): self._cleanup(instance) done.send() - greenthread.spawn(_wait_for_time) + greenthread.spawn(_wait_for_timer) return done def _cleanup(self, instance): @@ -365,9 +365,9 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): logging.info('cool, it\'s a device') - r = utils.execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) - return r[0] + out, err = utils.execute("sudo dd if=%s iflag=nonblock" % + virsh_output, check_exit_code=False) + return out else: return '' @@ -388,8 +388,7 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo chown %d %s' % (os.getuid(), - console_log)) + utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) if FLAGS.libvirt_type == 'xen': # Xen is special @@ -476,7 +475,6 @@ class LibvirtConnection(object): ['local_gb'] * 1024 * 1024 * 1024) - resize = inst['instance_type'] != 'm1.tiny' resize = True if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': resize = False @@ -743,7 +741,7 @@ class NWFilterFirewall(object): if callable(xml): xml = xml() - # execute in a native thread and block until done + # execute in a native thread and block current greenthread until done tpool.execute(self._conn.nwfilterDefineXML, xml) @staticmethod diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xenapi/network_utils.py index d8632f3939..0129543944 100644 --- a/nova/virt/xenapi/network_utils.py +++ b/nova/virt/xenapi/network_utils.py @@ -30,10 +30,9 @@ class NetworkHelper(): @classmethod def find_network_with_bridge(cls, session, bridge): - """ Return the network on which the bridge is attached, if found """ + """ Return the network on which the bridge is attached, if found.""" expr = 'field "bridge" = "%s"' % bridge - networks = session.call_xenapi('network.get_all_records_where', - expr) + networks = session.call_xenapi('network.get_all_records_where', expr) if len(networks) == 1: return networks.keys()[0] elif len(networks) > 1: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b6b92b9264..3034df9e1d 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -107,8 +107,7 @@ class VMOps(object): if vdis: for vdi in vdis: try: - task = self._session.call_xenapi('Async.VDI.destroy', - vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi) self._session.wait_for_task(task) except XenAPI.Failure, exc: logging.warn(exc) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index df8e42d347..4243111330 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -175,9 +175,11 @@ class XenAPISession(object): The task is polled until it completes.""" done = event.Event() - loop = utis.LoopingTask(self._poll_task, task, done) + loop = utils.LoopingTask(self._poll_task, task, done) loop.start(FLAGS.xenapi_task_poll_interval, now=True) - return done.wait() + rv = done.wait() + loop.stop() + return rv def _poll_task(self, task, done): """Poll the given XenAPI task, and fire the given Deferred if we diff --git a/nova/volume/driver.py b/nova/volume/driver.py index f675c91320..1cd4c1fd45 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -22,10 +22,10 @@ Drivers for volumes. import logging import os +import time from nova import exception from nova import flags -from nova import process from nova import utils @@ -75,7 +75,7 @@ class VolumeDriver(object): raise logging.exception("Recovering from a failed execute." "Try number %s", tries) - self._execute("sleep %s" % tries ** 2) + time.sleep(tries ** 2) def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" @@ -91,21 +91,20 @@ class VolumeDriver(object): sizestr = '%sG' % volume['size'] self._try_execute("sudo lvcreate -L %s -n %s %s" % (sizestr, - volume['name'], - FLAGS.volume_group)) + volume['name'], + FLAGS.volume_group)) def delete_volume(self, volume): """Deletes a logical volume.""" self._try_execute("sudo lvremove -f %s/%s" % (FLAGS.volume_group, - volume['name'])) + volume['name'])) def local_path(self, volume): # NOTE(vish): stops deprecation warning escaped_group = FLAGS.volume_group.replace('-', '--') escaped_name = volume['name'].replace('-', '--') - return "/dev/mapper/%s-%s" % (escaped_group, - escaped_name) + return "/dev/mapper/%s-%s" % (escaped_group, escaped_name) def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" @@ -165,7 +164,7 @@ class AOEDriver(VolumeDriver): # still works for the other volumes, so we # just wait a bit for the current volume to # be ready and ignore any errors. - self._execute("sleep 2") + time.sleep(2) self._execute("sudo vblade-persist auto all", check_exit_code=False) self._execute("sudo vblade-persist start all", @@ -275,9 +274,8 @@ class ISCSIDriver(VolumeDriver): def discover_volume(self, volume): """Discover volume on a remote host.""" - (iscsi_name, - iscsi_portal) = self._get_name_and_portal(volume['name'], - volume['host']) + iscsi_name, iscsi_portal = self._get_name_and_portal(volume['name'], + volume['host']) self._execute("sudo iscsiadm -m node -T %s -p %s --login" % (iscsi_name, iscsi_portal)) self._execute("sudo iscsiadm -m node -T %s -p %s --op update " @@ -287,9 +285,8 @@ class ISCSIDriver(VolumeDriver): def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" - (iscsi_name, - iscsi_portal) = self._get_name_and_portal(volume['name'], - volume['host']) + iscsi_name, iscsi_portal = self._get_name_and_portal(volume['name'], + volume['host']) self._execute("sudo iscsiadm -m node -T %s -p %s --op update " "-n node.startup -v manual" % (iscsi_name, iscsi_portal)) diff --git a/run_tests.py b/run_tests.py index 6d7830a297..9f37084120 100644 --- a/run_tests.py +++ b/run_tests.py @@ -46,7 +46,6 @@ import __main__ import os import sys - from twisted.scripts import trial as trial_script from nova import flags @@ -66,8 +65,6 @@ from nova.tests.rpc_unittest import * from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * from nova.tests.twistd_unittest import * -from nova.tests.validator_unittest import * -from nova.tests.virt_unittest import * from nova.tests.virt_unittest import * from nova.tests.volume_unittest import * @@ -85,8 +82,6 @@ if __name__ == '__main__': config = OptionsClass() argv = config.parseOptions() - argv = FLAGS(sys.argv) - FLAGS.verbose = True # TODO(termie): these should make a call instead of doing work on import From aff411d80f4243ad0b40649af3a7586c7c38ac2d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 15 Dec 2010 11:57:56 +0100 Subject: [PATCH 112/119] Make sure we unlock the bzr tree again in the authors unit test. --- nova/tests/misc_unittest.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/nova/tests/misc_unittest.py b/nova/tests/misc_unittest.py index 667c63ad09..2758276e5b 100644 --- a/nova/tests/misc_unittest.py +++ b/nova/tests/misc_unittest.py @@ -30,23 +30,26 @@ class ProjectTestCase(test.TrialTestCase): import bzrlib.workingtree tree = bzrlib.workingtree.WorkingTree.open('..') tree.lock_read() - parents = tree.get_parent_ids() - g = tree.branch.repository.get_graph() - for p in parents[1:]: - rev_ids = [r for r, _ in g.iter_ancestry(parents) - if r != "null:"] - revs = tree.branch.repository.get_revisions(rev_ids) - for r in revs: - for author in r.get_apparent_authors(): - email = author.split(' ')[-1] - contributors.add(str_dict_replace(email, mailmap)) + try: + parents = tree.get_parent_ids() + g = tree.branch.repository.get_graph() + for p in parents[1:]: + rev_ids = [r for r, _ in g.iter_ancestry(parents) + if r != "null:"] + revs = tree.branch.repository.get_revisions(rev_ids) + for r in revs: + for author in r.get_apparent_authors(): + email = author.split(' ')[-1] + contributors.add(str_dict_replace(email, mailmap)) - authors_file = open('../Authors', 'r').read() + authors_file = open('../Authors', 'r').read() - missing = set() - for contributor in contributors: - if not contributor in authors_file: - missing.add(contributor) + missing = set() + for contributor in contributors: + if not contributor in authors_file: + missing.add(contributor) - self.assertTrue(len(missing) == 0, - '%r not listed in Authors' % missing) + self.assertTrue(len(missing) == 0, + '%r not listed in Authors' % missing) + finally: + tree.unlock() From 17daec6992456efc70ffbf05423ea91123db1fc2 Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Wed, 15 Dec 2010 20:17:44 +0300 Subject: [PATCH 113/119] Restore alphabetical order in Authors file --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index 2810a1ddb9..749d784061 100644 --- a/Authors +++ b/Authors @@ -5,6 +5,7 @@ Armando Migliaccio Chris Behrens Dean Troyer Devin Carlen +Eldar Nugaev Eric Day Ewan Mellor Hisaki Ohara @@ -28,4 +29,3 @@ Trey Morris Vishvananda Ishaya Youcef Laribi Zhixue Wu -Eldar Nugaev \ No newline at end of file From fdf067037981c2b4b4501258919af0f9e1d0ec26 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 15 Dec 2010 10:38:30 -0800 Subject: [PATCH 114/119] add missing import --- nova/virt/xenapi_conn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 4243111330..a88101ad02 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -51,6 +51,8 @@ import logging import sys import xmlrpclib +from eventlet import event + from nova import utils from nova import flags from nova.virt.xenapi.vmops import VMOps From 9a8113584edc9a8dbf42e7039b373429c11a7760 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 15 Dec 2010 11:53:18 -0800 Subject: [PATCH 115/119] fixes for xenapi (thanks sandywalsh) --- nova/virt/xenapi_conn.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index a88101ad02..09d399da4d 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -52,6 +52,7 @@ import sys import xmlrpclib from eventlet import event +from eventlet import tpool from nova import utils from nova import flags @@ -164,20 +165,20 @@ class XenAPISession(object): f = self._session.xenapi for m in method.split('.'): f = f.__getattr__(m) - return f(*args) + return tpool.execute(f, *args) def async_call_plugin(self, plugin, fn, args): """Call Async.host.call_plugin on a background thread.""" - return _unwrap_plugin_exceptions( - self._session.xenapi.Async.host.call_plugin, - self.get_xenapi_host(), plugin, fn, args) + return tpool.execute(_unwrap_plugin_exceptions, + self._session.xenapi.Async.host.call_plugin, + self.get_xenapi_host(), plugin, fn, args) def wait_for_task(self, task): """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" done = event.Event() - loop = utils.LoopingTask(self._poll_task, task, done) + loop = utils.LoopingCall(self._poll_task, task, done) loop.start(FLAGS.xenapi_task_poll_interval, now=True) rv = done.wait() loop.stop() From 9b049acc27d477a1ab9e13c9e064e59d8bd0a3ae Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 16 Dec 2010 10:52:30 -0800 Subject: [PATCH 116/119] pep8 fixes --- nova/compute/manager.py | 2 +- nova/service.py | 10 +++++----- nova/test.py | 1 + nova/tests/rpc_unittest.py | 2 +- nova/tests/service_unittest.py | 2 +- nova/utils.py | 7 ++++--- nova/virt/libvirt_conn.py | 12 ++++++------ nova/virt/xenapi_conn.py | 2 +- 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index f90f28b78d..7eb60e262d 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -64,7 +64,7 @@ class ComputeManager(manager.Manager): self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(*args, **kwargs) - + def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? diff --git a/nova/service.py b/nova/service.py index 3d40e83a6c..ac30aaceba 100644 --- a/nova/service.py +++ b/nova/service.py @@ -99,7 +99,7 @@ class Service(object): self.timers.append(consumer_all.attach_to_eventlet()) self.timers.append(consumer_node.attach_to_eventlet()) - + pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) self.timers.append(pulse) @@ -122,7 +122,7 @@ class Service(object): return getattr(manager, key) @classmethod - def create(cls, + def create(cls, host=None, binary=None, topic=None, @@ -192,7 +192,7 @@ class Service(object): db.service_update(ctxt, self.service_id, {'report_count': service_ref['report_count'] + 1}) - + # TODO(termie): make this pattern be more elegant. if getattr(self, "model_disconnected", False): self.model_disconnected = False @@ -207,7 +207,7 @@ class Service(object): def serve(*services): argv = FLAGS(sys.argv) - + if not services: services = [Service.create()] @@ -227,7 +227,7 @@ def serve(*services): for x in services: x.start() - + def wait(): while True: diff --git a/nova/test.py b/nova/test.py index ecc97aa4d9..7076f1bf4e 100644 --- a/nova/test.py +++ b/nova/test.py @@ -55,6 +55,7 @@ def skip_if_fake(func): return func(*args, **kw) return _skipper + class TestCase(unittest.TestCase): """Test case base class for all unit tests""" def setUp(self): diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index 4128c35b8c..a2495e65a2 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -67,7 +67,7 @@ class RpcTestCase(test.TestCase): self.assertRaises(rpc.RemoteError, rpc.call, self.context, - 'test', + 'test', {"method": "fail", "args": {"value": value}}) try: diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 6bdc7071cb..47c092f8e2 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -119,7 +119,7 @@ class ServiceTestCase(test.TestCase): service.db.service_create(mox.IgnoreArg(), service_create).AndReturn(service_ref) self.mox.ReplayAll() - + app.start() app.stop() self.assert_(app) diff --git a/nova/utils.py b/nova/utils.py index 5f2d472021..ea1f04ca79 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -233,6 +233,7 @@ class LoopingCall(object): def start(self, interval, now=True): self._running = True done = event.Event() + def _inner(): if not now: greenthread.sleep(interval) @@ -244,14 +245,14 @@ class LoopingCall(object): logging.exception('in looping call') done.send_exception(*sys.exc_info()) return - + done.send(True) self.done = done - + greenthread.spawn(_inner) return self.done - + def stop(self): self._running = False diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index ba51f8f698..5a8c718506 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -205,7 +205,7 @@ class LibvirtConnection(object): timer.f = _wait_for_shutdown timer_done = timer.start(interval=0.5, now=True) - + # NOTE(termie): this is strictly superfluous (we could put the # cleanup code in the timer), but this emulates the # previous model so I am keeping it around until @@ -387,7 +387,7 @@ class LibvirtConnection(object): def get_console_output(self, instance): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - + utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) if FLAGS.libvirt_type == 'xen': @@ -439,11 +439,11 @@ class LibvirtConnection(object): if not os.path.exists(basepath('ramdisk')): images.fetch(inst.ramdisk_id, basepath('ramdisk'), user, project) - + def execute(cmd, process_input=None, check_exit_code=True): - return utils.execute(cmd=cmd, - process_input=process_input, - check_exit_code=check_exit_code) + return utils.execute(cmd=cmd, + process_input=process_input, + check_exit_code=check_exit_code) key = str(inst['key_data']) net = None diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 09d399da4d..6beb08f5e8 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -200,7 +200,7 @@ class XenAPISession(object): error_info = self._session.xenapi.task.get_error_info(task) logging.warn('Task %s status: %s. %s', task, status, error_info) - done.send_exception(XenAPI.Failure(error_info)) + done.send_exception(XenAPI.Failure(error_info)) #logging.debug('Polling task %s done.', task) except XenAPI.Failure, exc: logging.warn(exc) From e306d236e3cf91c00a36940e277ad2d105b055a1 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 16 Dec 2010 12:45:06 -0800 Subject: [PATCH 117/119] pep8 fixes for bin --- bin/nova-combined | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/nova-combined b/bin/nova-combined index 5f635b3a3d..c6a04f7e93 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -50,7 +50,7 @@ flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) - + compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') volume = service.Service.create(binary='nova-volume') @@ -63,4 +63,3 @@ if __name__ == '__main__': server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) server.wait() - From 611935aa3e3a66e9638b0c127041a6fca4788b9c Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 16 Dec 2010 15:03:37 -0600 Subject: [PATCH 118/119] Put flags back in vm_utils --- nova/virt/xenapi/vm_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index dde1384048..b83ae94751 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -21,15 +21,20 @@ their attributes like VDIs, VIFs, as well as their lookup functions. import logging import urllib + from xml.dom import minidom +from nova import flags from nova import utils + from nova.auth.manager import AuthManager from nova.compute import instance_types from nova.compute import power_state from nova.virt import images +FLAGS = flags.FLAGS + XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, 'Running': power_state.RUNNING, @@ -37,7 +42,6 @@ XENAPI_POWER_STATE = { 'Suspended': power_state.SHUTDOWN, # FIXME 'Crashed': power_state.CRASHED} - XenAPI = None From e5a3d993cb13c8dc5e984a67521f77ce8fdf8e4c Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 16 Dec 2010 15:19:35 -0600 Subject: [PATCH 119/119] Removed unnecessary blank lines --- nova/virt/xenapi/vm_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index b83ae94751..2f5d78e759 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -21,12 +21,10 @@ their attributes like VDIs, VIFs, as well as their lookup functions. import logging import urllib - from xml.dom import minidom from nova import flags from nova import utils - from nova.auth.manager import AuthManager from nova.compute import instance_types from nova.compute import power_state