Merge trunk

Authors

@@ -56,6 +56,7 @@ Thierry Carrez <thierry@openstack.org>
 Todd Willey <todd@ansolabs.com>
 Trey Morris <trey.morris@rackspace.com>
 Tushar Patil <tushar.vitthal.patil@gmail.com> <tpatil@vertex.co.in>
+Vasiliy Shlykov <vash@vasiliyshlykov.org>
 Vishvananda Ishaya <vishvananda@gmail.com>
 Youcef Laribi <Youcef.Laribi@eu.citrix.com>
 Zhixue Wu <Zhixue.Wu@citrix.com>

MANIFEST.in

@@ -6,14 +6,23 @@ graft doc
 graft smoketests
 graft tools
 graft etc
+graft bzrplugins
+graft contrib
+graft po
+graft plugins
 include nova/api/openstack/notes.txt
+include nova/auth/*.schema
 include nova/auth/novarc.template
+include nova/auth/opendj.sh
 include nova/auth/slap.sh
 include nova/cloudpipe/bootscript.sh
 include nova/cloudpipe/client.ovpn.template
+include nova/cloudpipe/bootscript.template
 include nova/compute/fakevirtinstance.xml
 include nova/compute/interfaces.template
+include nova/console/xvp.conf.template
 include nova/db/sqlalchemy/migrate_repo/migrate.cfg
+include nova/db/sqlalchemy/migrate_repo/README
 include nova/virt/interfaces.template
 include nova/virt/libvirt*.xml.template
 include nova/tests/CA/
@@ -25,6 +34,7 @@ include nova/tests/bundle/1mb.manifest.xml
 include nova/tests/bundle/1mb.no_kernel_or_ramdisk.manifest.xml
 include nova/tests/bundle/1mb.part.0
 include nova/tests/bundle/1mb.part.1
+include nova/tests/db/nova.austin.sqlite
 include plugins/xenapi/README
 include plugins/xenapi/etc/xapi.d/plugins/objectstore
 include plugins/xenapi/etc/xapi.d/plugins/pluginlib_nova.py

locale/nova.pot

@@ -1826,7 +1826,7 @@ msgstr ""
 
 #: nova/virt/xenapi/vm_utils.py:290
 #, python-format
-msgid "PV Kernel in VDI:%d"
+msgid "PV Kernel in VDI:%s"
 msgstr ""
 
 #: nova/virt/xenapi/vm_utils.py:318

nova/api/ec2/cloud.py

@@ -883,6 +883,9 @@ class CloudController(object):
                                      % attribute)
         try:
             image = self.image_service.show(context, image_id)
+            image = self._format_image(context,
+                                       self.image_service.show(context,
+                                                               image_id))
         except IndexError:
             raise exception.ApiError(_('invalid id: %s') % image_id)
         result = {'image_id': image_id, 'launchPermission': []}

nova/compute/api.py

@@ -67,10 +67,10 @@ class API(base.Base):
         """Get the network topic for an instance."""
         try:
             instance = self.get(context, instance_id)
-        except exception.NotFound as e:
+        except exception.NotFound:
             LOG.warning(_("Instance %d was not found in get_network_topic"),
                         instance_id)
-            raise e
+            raise
 
         host = instance['host']
         if not host:
@@ -103,9 +103,9 @@ class API(base.Base):
         if not is_vpn:
             image = self.image_service.show(context, image_id)
             if kernel_id is None:
-                kernel_id = image.get('kernelId', None)
+                kernel_id = image.get('kernel_id', None)
             if ramdisk_id is None:
-                ramdisk_id = image.get('ramdiskId', None)
+                ramdisk_id = image.get('ramdisk_id', None)
             # No kernel and ramdisk for raw images
             if kernel_id == str(FLAGS.null_kernel):
                 kernel_id = None
@@ -293,10 +293,10 @@ class API(base.Base):
         LOG.debug(_("Going to try to terminate %s"), instance_id)
         try:
             instance = self.get(context, instance_id)
-        except exception.NotFound as e:
+        except exception.NotFound:
             LOG.warning(_("Instance %d was not found during terminate"),
                         instance_id)
-            raise e
+            raise
 
         if (instance['state_description'] == 'terminating'):
             LOG.warning(_("Instance %d is already being terminated"),

nova/compute/manager.py

@@ -127,7 +127,7 @@ class ComputeManager(manager.Manager):
             info = self.driver.get_info(instance_ref['name'])
             state = info['state']
         except exception.NotFound:
-            state = power_state.NOSTATE
+            state = power_state.FAILED
         self.db.instance_set_state(context, instance_id, state)
 
     def get_console_topic(self, context, **_kwargs):

nova/compute/power_state.py

@@ -27,6 +27,7 @@ SHUTDOWN = 0x04
 SHUTOFF = 0x05
 CRASHED = 0x06
 SUSPENDED = 0x07
+FAILED = 0x08
 
 
 def name(code):
@@ -38,5 +39,6 @@ def name(code):
         SHUTDOWN: 'shutdown',
         SHUTOFF: 'shutdown',
         CRASHED: 'crashed',
-        SUSPENDED: 'suspended'}
+        SUSPENDED: 'suspended',
+        FAILED: 'failed to spawn'}
     return d[code]

nova/context.py

@@ -28,7 +28,6 @@ from nova import utils
 
 
 class RequestContext(object):
-
     def __init__(self, user, project, is_admin=None, read_deleted=False,
                  remote_address=None, timestamp=None, request_id=None):
         if hasattr(user, 'id'):
@@ -53,7 +52,7 @@ class RequestContext(object):
         self.read_deleted = read_deleted
         self.remote_address = remote_address
         if not timestamp:
-            timestamp = datetime.datetime.utcnow()
+            timestamp = utils.utcnow()
         if isinstance(timestamp, str) or isinstance(timestamp, unicode):
             timestamp = utils.parse_isotime(timestamp)
         self.timestamp = timestamp
@@ -101,7 +100,7 @@ class RequestContext(object):
         return cls(**values)
 
     def elevated(self, read_deleted=False):
-        """Return a version of this context with admin flag set"""
+        """Return a version of this context with admin flag set."""
         return RequestContext(self.user_id,
                               self.project_id,
                               True,

nova/db/sqlalchemy/migrate_repo/versions/001_austin.py

@@ -508,17 +508,19 @@ def upgrade(migrate_engine):
     # bind migrate_engine to your metadata
     meta.bind = migrate_engine
 
-    for table in (auth_tokens, export_devices, fixed_ips, floating_ips,
-                  instances, key_pairs, networks,
-                  projects, quotas, security_groups, security_group_inst_assoc,
-                  security_group_rules, services, users,
-                  user_project_association, user_project_role_association,
-                  user_role_association, volumes):
+    tables = [auth_tokens,
+              instances, key_pairs, networks, fixed_ips, floating_ips,
+              quotas, security_groups, security_group_inst_assoc,
+              security_group_rules, services, users, projects,
+              user_project_association, user_project_role_association,
+              user_role_association, volumes, export_devices]
+    for table in tables:
         try:
             table.create()
         except Exception:
             logging.info(repr(table))
             logging.exception('Exception while creating table')
+            meta.drop_all(tables=tables)
             raise
 
 

nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py

@@ -209,13 +209,16 @@ def upgrade(migrate_engine):
     # Upgrade operations go here. Don't create your own engine;
     # bind migrate_engine to your metadata
     meta.bind = migrate_engine
-    for table in (certificates, consoles, console_pools, instance_actions,
-                  iscsi_targets):
+
+    tables = [certificates, console_pools, consoles, instance_actions,
+              iscsi_targets]
+    for table in tables:
         try:
             table.create()
         except Exception:
             logging.info(repr(table))
             logging.exception('Exception while creating table')
+            meta.drop_all(tables=tables)
             raise
 
     auth_tokens.c.user_id.alter(type=String(length=255,

nova/flags.py

@@ -282,6 +282,8 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger')
 
 DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'),
               "Top-level directory for maintaining nova's state")
+DEFINE_string('logdir', None, 'output to a per-service log file in named '
+                              'directory')
 
 DEFINE_string('sql_connection',
               'sqlite:///$state_path/nova.sqlite',

nova/image/s3.py

@@ -94,7 +94,7 @@ class S3ImageService(service.BaseImageService):
         if FLAGS.connection_type == 'fake':
             return {'imageId': 'bar'}
         result = self.index(context)
-        result = [i for i in result if i['imageId'] == image_id]
+        result = [i for i in result if i['id'] == image_id]
         if not result:
             raise exception.NotFound(_('Image %s could not be found')
                                      % image_id)

nova/log.py

@@ -28,9 +28,11 @@ It also allows setting of formatting information through flags.
 
 
 import cStringIO
+import inspect
 import json
 import logging
 import logging.handlers
+import os
 import sys
 import traceback
 
@@ -111,6 +113,18 @@ def _dictify_context(context):
     return context
 
 
+def _get_binary_name():
+    return os.path.basename(inspect.stack()[-1][1])
+
+
+def get_log_file_path(binary=None):
+    if FLAGS.logfile:
+        return FLAGS.logfile
+    if FLAGS.logdir:
+        binary = binary or _get_binary_name()
+        return '%s.log' % (os.path.join(FLAGS.logdir, binary),)
+
+
 def basicConfig():
     logging.basicConfig()
     for handler in logging.root.handlers:
@@ -123,8 +137,9 @@ def basicConfig():
         syslog = SysLogHandler(address='/dev/log')
         syslog.setFormatter(_formatter)
         logging.root.addHandler(syslog)
-    if FLAGS.logfile:
-        logfile = RotatingFileHandler(FLAGS.logfile)
+    logpath = get_log_file_path()
+    if logpath:
+        logfile = RotatingFileHandler(logpath)
         logfile.setFormatter(_formatter)
         logging.root.addHandler(logfile)
 

nova/network/linux_net.py

@@ -54,6 +54,8 @@ flags.DEFINE_string('routing_source_ip', '$my_ip',
                     'Public IP of network host')
 flags.DEFINE_bool('use_nova_chains', False,
                   'use the nova_ routing chains instead of default')
+flags.DEFINE_string('input_chain', 'INPUT',
+                    'chain to add nova_input to')
 
 flags.DEFINE_string('dns_server', None,
                     'if set, uses specific dns server for dnsmasq')

nova/tests/test_api.py

@@ -248,16 +248,14 @@ class ApiEc2TestCase(test.TestCase):
         self.mox.ReplayAll()
 
         rv = self.ec2.get_all_security_groups()
-        # I don't bother checkng that we actually find it here,
-        # because the create/delete unit test further up should
-        # be good enough for that.
-        for group in rv:
-            if group.name == security_group_name:
-                self.assertEquals(len(group.rules), 1)
-                self.assertEquals(int(group.rules[0].from_port), 80)
-                self.assertEquals(int(group.rules[0].to_port), 81)
-                self.assertEquals(len(group.rules[0].grants), 1)
-                self.assertEquals(str(group.rules[0].grants[0]), '0.0.0.0/0')
+
+        group = [grp for grp in rv if grp.name == security_group_name][0]
+
+        self.assertEquals(len(group.rules), 1)
+        self.assertEquals(int(group.rules[0].from_port), 80)
+        self.assertEquals(int(group.rules[0].to_port), 81)
+        self.assertEquals(len(group.rules[0].grants), 1)
+        self.assertEquals(str(group.rules[0].grants[0]), '0.0.0.0/0')
 
         self.expect_http()
         self.mox.ReplayAll()
@@ -314,16 +312,13 @@ class ApiEc2TestCase(test.TestCase):
         self.mox.ReplayAll()
 
         rv = self.ec2.get_all_security_groups()
-        # I don't bother checkng that we actually find it here,
-        # because the create/delete unit test further up should
-        # be good enough for that.
-        for group in rv:
-            if group.name == security_group_name:
-                self.assertEquals(len(group.rules), 1)
-                self.assertEquals(int(group.rules[0].from_port), 80)
-                self.assertEquals(int(group.rules[0].to_port), 81)
-                self.assertEquals(len(group.rules[0].grants), 1)
-                self.assertEquals(str(group.rules[0].grants[0]), '::/0')
+
+        group = [grp for grp in rv if grp.name == security_group_name][0]
+        self.assertEquals(len(group.rules), 1)
+        self.assertEquals(int(group.rules[0].from_port), 80)
+        self.assertEquals(int(group.rules[0].to_port), 81)
+        self.assertEquals(len(group.rules[0].grants), 1)
+        self.assertEquals(str(group.rules[0].grants[0]), '::/0')
 
         self.expect_http()
         self.mox.ReplayAll()

nova/tests/test_log.py

@@ -46,6 +46,27 @@ class RootLoggerTestCase(test.TestCase):
         self.assert_(True)  # didn't raise exception
 
 
+class LogHandlerTestCase(test.TestCase):
+    def test_log_path_logdir(self):
+        self.flags(logdir='/some/path')
+        self.assertEquals(log.get_log_file_path(binary='foo-bar'),
+                         '/some/path/foo-bar.log')
+
+    def test_log_path_logfile(self):
+        self.flags(logfile='/some/path/foo-bar.log')
+        self.assertEquals(log.get_log_file_path(binary='foo-bar'),
+                         '/some/path/foo-bar.log')
+
+    def test_log_path_none(self):
+        self.assertTrue(log.get_log_file_path(binary='foo-bar') is None)
+
+    def test_log_path_logfile_overrides_logdir(self):
+        self.flags(logdir='/some/other/path',
+                   logfile='/some/path/foo-bar.log')
+        self.assertEquals(log.get_log_file_path(binary='foo-bar'),
+                         '/some/path/foo-bar.log')
+
+
 class NovaFormatterTestCase(test.TestCase):
     def setUp(self):
         super(NovaFormatterTestCase, self).setUp()

nova/tests/test_xenapi.py

@@ -243,7 +243,8 @@ class XenAPIVMTestCase(test.TestCase):
         # Check that the VM is running according to XenAPI.
         self.assertEquals(vm['power_state'], 'Running')
 
-    def _test_spawn(self, image_id, kernel_id, ramdisk_id):
+    def _test_spawn(self, image_id, kernel_id, ramdisk_id,
+                    instance_type="m1.large"):
         stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests)
         values = {'name': 1,
                   'id': 1,
@@ -252,7 +253,7 @@ class XenAPIVMTestCase(test.TestCase):
                   'image_id': image_id,
                   'kernel_id': kernel_id,
                   'ramdisk_id': ramdisk_id,
-                  'instance_type': 'm1.large',
+                  'instance_type': instance_type,
                   'mac_address': 'aa:bb:cc:dd:ee:ff',
                   }
         conn = xenapi_conn.get_connection(False)
@@ -260,6 +261,12 @@ class XenAPIVMTestCase(test.TestCase):
         conn.spawn(instance)
         self.check_vm_record(conn)
 
+    def test_spawn_not_enough_memory(self):
+        FLAGS.xenapi_image_service = 'glance'
+        self.assertRaises(Exception,
+                          self._test_spawn,
+                          1, 2, 3, "m1.xlarge")
+
     def test_spawn_raw_objectstore(self):
         FLAGS.xenapi_image_service = 'objectstore'
         self._test_spawn(1, None, None)

nova/twistd.py

@@ -43,8 +43,6 @@ else:
 
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('logdir', None, 'directory to keep log files in '
-                                     '(will be prepended to $logfile)')
 
 
 class TwistdServerOptions(ServerOptions):

nova/utils.py

@@ -25,7 +25,6 @@ import inspect
 import json
 import os
 import random
-import subprocess
 import socket
 import struct
 import sys
@@ -36,6 +35,7 @@ import netaddr
 
 from eventlet import event
 from eventlet import greenthread
+from eventlet.green import subprocess
 
 from nova import exception
 from nova.exception import ProcessExecutionError

nova/virt/xenapi/fake.py

@@ -286,6 +286,10 @@ class SessionBase(object):
         rec['currently_attached'] = False
         rec['device'] = ''
 
+    def host_compute_free_memory(self, _1, ref):
+        #Always return 12GB available
+        return 12 * 1024 * 1024 * 1024
+
     def xenapi_request(self, methodname, params):
         if methodname.startswith('login'):
             self._login(methodname, params)

nova/virt/xenapi/vm_utils.py

@@ -138,6 +138,16 @@ class VMHelper(HelperBase):
         LOG.debug(_('Created VM %(instance_name)s as %(vm_ref)s.') % locals())
         return vm_ref
 
+    @classmethod
+    def ensure_free_mem(cls, session, instance):
+        instance_type = instance_types.INSTANCE_TYPES[instance.instance_type]
+        mem = long(instance_type['memory_mb']) * 1024 * 1024
+        #get free memory from host
+        host = session.get_xenapi_host()
+        host_free_mem = long(session.get_xenapi().host.
+                             compute_free_memory(host))
+        return host_free_mem >= mem
+
     @classmethod
     def create_vbd(cls, session, vm_ref, vdi_ref, userdevice, bootable):
         """Create a VBD record.  Returns a Deferred that gives the new
@@ -384,7 +394,7 @@ class VMHelper(HelperBase):
             pv = True
         elif pv_str.lower() == 'false':
             pv = False
-        LOG.debug(_("PV Kernel in VDI:%d"), pv)
+        LOG.debug(_("PV Kernel in VDI:%s"), pv)
         return pv
 
     @classmethod

nova/virt/xenapi/vmops.py

@@ -66,7 +66,15 @@ class VMOps(object):
         if vm is not None:
             raise exception.Duplicate(_('Attempted to create'
             ' non-unique name %s') % instance.name)
-
+        #ensure enough free memory is available
+        if not VMHelper.ensure_free_mem(self._session, instance):
+                name = instance['name']
+                LOG.exception(_('instance %(name)s: not enough free memory')
+                              % locals())
+                db.instance_set_state(context.get_admin_context(),
+                                      instance['id'],
+                                      power_state.SHUTDOWN)
+                return
         bridge = db.network_get_by_instance(context.get_admin_context(),
                                             instance['id'])['bridge']
         network_ref = \
@@ -161,7 +169,8 @@ class VMOps(object):
                 instance_name = instance_or_vm.name
         vm = VMHelper.lookup(self._session, instance_name)
         if vm is None:
-            raise Exception(_('Instance not present %s') % instance_name)
+            raise exception.NotFound(
+                            _('Instance not present %s') % instance_name)
         return vm
 
     def snapshot(self, instance, image_id):

nova/volume/manager.py

@@ -111,10 +111,10 @@ class VolumeManager(manager.Manager):
 
             LOG.debug(_("volume %s: creating export"), volume_ref['name'])
             self.driver.create_export(context, volume_ref)
-        except Exception as e:
+        except Exception:
             self.db.volume_update(context,
                                   volume_ref['id'], {'status': 'error'})
-            raise e
+            raise
 
         now = datetime.datetime.utcnow()
         self.db.volume_update(context,
@@ -137,11 +137,11 @@ class VolumeManager(manager.Manager):
             self.driver.remove_export(context, volume_ref)
             LOG.debug(_("volume %s: deleting"), volume_ref['name'])
             self.driver.delete_volume(volume_ref)
-        except Exception as e:
+        except Exception:
             self.db.volume_update(context,
                                   volume_ref['id'],
                                   {'status': 'error_deleting'})
-            raise e
+            raise
 
         self.db.volume_destroy(context, volume_id)
         LOG.debug(_("volume %s: deleted successfully"), volume_ref['name'])

setup.py

@@ -85,9 +85,13 @@ setup(name='nova',
       packages=find_packages(exclude=['bin', 'smoketests']),
       include_package_data=True,
       test_suite='nose.collector',
-      scripts=['bin/nova-api',
+      scripts=['bin/nova-ajax-console-proxy',
+               'bin/nova-api',
+               'bin/nova-combined',
                'bin/nova-compute',
+               'bin/nova-console',
                'bin/nova-dhcpbridge',
+               'bin/nova-direct-api',
                'bin/nova-import-canonical-imagestore',
                'bin/nova-instancemonitor',
                'bin/nova-logspool',
@@ -96,5 +100,6 @@ setup(name='nova',
                'bin/nova-objectstore',
                'bin/nova-scheduler',
                'bin/nova-spoolsentry',
+               'bin/stack',
                'bin/nova-volume',
                'tools/nova-debug'])