From 80a6dc5504378ae3d96829d96c02f50b9daa3029 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 9 Mar 2011 13:46:05 -0600 Subject: [PATCH 001/101] stuff --- nova/compute/api.py | 15 +++++++++++++-- nova/compute/manager.py | 17 ++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 33d25fc4bf..93f0a12c13 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -456,12 +456,23 @@ class API(base.Base): self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) - def resize(self, context, instance_id, flavor): + def resize(self, context, instance_id, flavor_id): """Resize a running instance.""" + instance = self.db.instance_get(context, instance_id) + current_instance_type = self.db.instance_type_get_by_flavor_id( + context, instance['flavor_id']) + new_instance_type = self.db.instance_type_get_by_flavor_id( + context, flavor_id) + + if current_instance_type.memory_mb > new_instance_typ.memory_mb: + raise exception.ApiError(_("Invalid flavor: cannot downsize" + "instances")) + self._cast_scheduler_message(context, {"method": "prep_resize", "args": {"topic": FLAGS.compute_topic, - "instance_id": instance_id, }},) + "instance_id": instance_id, + "instance_type": new_instance_type}}) def pause(self, context, instance_id): """Pause the given instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b3e864154a..f85ad91dfe 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -447,7 +447,7 @@ class ComputeManager(manager.Manager): @exception.wrap_exception @checks_instance_lock - def prep_resize(self, context, instance_id): + def prep_resize(self, context, instance_id, flavor_id): """Initiates the process of moving a running instance to another host, possibly changing the RAM and disk size in the process""" context = context.elevated() @@ -456,12 +456,17 @@ class ComputeManager(manager.Manager): raise exception.Error(_( 'Migration error: destination same as source!')) + instance_type = self.db.instance_type_get_by_flavor_id(context, + flavor_id) migration_ref = self.db.migration_create(context, {'instance_id': instance_id, 'source_compute': instance_ref['host'], 'dest_compute': FLAGS.host, 'dest_host': self.driver.get_host_ip_addr(), + 'old_flavor': instance_type['flavor_id'], + 'new_flavor': flavor_id, 'status': 'pre-migrating'}) + LOG.audit(_('instance %s: migrating to '), instance_id, context=context) topic = self.db.queue_get_for(context, FLAGS.compute_topic, @@ -487,8 +492,14 @@ class ComputeManager(manager.Manager): self.db.migration_update(context, migration_id, {'status': 'post-migrating', }) - #TODO(mdietz): This is where we would update the VM record - #after resizing + #TODO(mdietz): apply the rest of the instance_type attributes going + #after they're supported + self.db.instance_update(context, instance_ref, + dict(memory_mb=instance_type['memory_mb'], + vcpus=instance_type['vcpus'], + local_gb=instance_type['local_gb'])) + self.driver.resize_instance(context, instance_ref) + service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, From 4a9f4f4eef4e6fd6ab84ec2e03437144f9ab62f8 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Mar 2011 16:07:50 -0600 Subject: [PATCH 002/101] More resize --- nova/compute/manager.py | 68 +++++++++++++++++++++++++-------------- nova/virt/xenapi/vmops.py | 13 ++++++-- nova/virt/xenapi_conn.py | 9 +++--- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 399356a13d..f73e813452 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -429,21 +429,37 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_get(context, migration_id) - #TODO(mdietz): we may want to split these into separate methods. - if migration_ref['source_compute'] == FLAGS.host: - self.driver._start(instance_ref) - self.db.migration_update(context, migration_id, - {'status': 'reverted'}) - else: - self.driver.destroy(instance_ref) - topic = self.db.queue_get_for(context, FLAGS.compute_topic, - instance_ref['host']) - rpc.cast(context, topic, - {'method': 'revert_resize', - 'args': { - 'migration_id': migration_ref['id'], - 'instance_id': instance_id, }, - }) + self.driver.destroy(instance_ref) + topic = self.db.queue_get_for(context, FLAGS.compute_topic, + instance_ref['host']) + rpc.cast(context, topic, + {'method': 'finish_revert_resize', + 'args': { + 'migration_id': migration_ref['id'], + 'instance_id': instance_id, }, + }) + + @exception.wrap_exception + @checks_instance_lock + def finish_revert_resize(self, context, instance_id, migration_id): + """Finishes the second half of reverting a resize, powering back on + the source instance and reverting the resized attributes in the + database""" + instance_ref = self.db.instance_get(context, instance_id) + migration_ref = self.db.migration_get(context, migration_id) + instance_type = self.db.instance_type_get_by_flavor_id(context, + migration_ref['old_flavor_id']) + + #Just roll back the record. There's no need to resize down since + #the 'old' VM already has the preferred attributes + self.db.instance_update(context, + dict(memory_mb=instance_type['memory_mb'], + vcpus=instance_type['vcpus'], + local_gb=instance_type['local_gb'])) + + self.driver._start(instance_ref) + self.db.migration_update(context, migration_id, + {'status': 'reverted'}) @exception.wrap_exception @checks_instance_lock @@ -463,8 +479,8 @@ class ComputeManager(manager.Manager): 'source_compute': instance_ref['host'], 'dest_compute': FLAGS.host, 'dest_host': self.driver.get_host_ip_addr(), - 'old_flavor': instance_type['flavor_id'], - 'new_flavor': flavor_id, + 'old_flavor_id': instance_type['flavor_id'], + 'new_flavor_id': flavor_id, 'status': 'pre-migrating'}) LOG.audit(_('instance %s: migrating to '), instance_id, @@ -492,13 +508,7 @@ class ComputeManager(manager.Manager): self.db.migration_update(context, migration_id, {'status': 'post-migrating', }) - #TODO(mdietz): apply the rest of the instance_type attributes going - #after they're supported - self.db.instance_update(context, instance_ref, - dict(memory_mb=instance_type['memory_mb'], - vcpus=instance_type['vcpus'], - local_gb=instance_type['local_gb'])) - self.driver.resize_instance(context, instance_ref) + service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_compute'], FLAGS.compute_topic) @@ -520,7 +530,17 @@ class ComputeManager(manager.Manager): migration_ref = self.db.migration_get(context, migration_id) instance_ref = self.db.instance_get(context, migration_ref['instance_id']) + + #TODO(mdietz): apply the rest of the instance_type attributes going + #after they're supported + instance_type = self.db.instance_type_get_by_flavor_id(context, + migration_ref['new_flavor_id']) + self.db.instance_update(context, instance_ref, + dict(memory_mb=instance_type['memory_mb'], + vcpus=instance_type['vcpus'], + local_gb=instance_type['local_gb'])) + self.driver.resize_instance(instance_ref, disk_info) self.driver.finish_resize(instance_ref, disk_info) self.db.migration_update(context, migration_id, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 562ecd4d55..9e0bd6a752 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -365,9 +365,18 @@ class VMOps(object): return new_cow_uuid - def resize(self, instance, flavor): + def resize_instance(self, instance, vdi_uuid): """Resize a running instance by changing it's RAM and disk size """ - raise NotImplementedError() + vm_ref = VMHelper.lookup(self._session, instance.name) + vdi_ref, vm_vdi_rec = \ + VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) + new_disk_size = instance.local_gb + + #TODO(mdietz): this will need to be adjusted for swap later + task = self._session.call_xenapi('VDI.resize_online', vdi_ref, + new_disk_size) + vm_ref = VMHelper.lookup(self._session, instance.name) + self._session.wait_for_task(task, instance.id) def reboot(self, instance): """Reboot VM instance""" diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index b63a5f8c3f..92b262479c 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -158,6 +158,11 @@ class XenAPIConnection(object): """Create VM instance""" self._vmops.spawn(instance) + def resize_instance(self, instance, disk_info): + """Resizes instance attributes such as RAM and disk space to the + attributes specified by the record""" + self._vmops.resize_instance(instance, disk_info['cow']) + def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" vdi_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], @@ -168,10 +173,6 @@ class XenAPIConnection(object): """ Create snapshot from a running VM instance """ self._vmops.snapshot(instance, image_id) - def resize(self, instance, flavor): - """Resize a VM instance""" - raise NotImplementedError() - def reboot(self, instance): """Reboot VM instance""" self._vmops.reboot(instance) From 6cd90a95d632d45d1c906d412e3240f730e88b95 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Mar 2011 15:35:55 -0600 Subject: [PATCH 003/101] New migration --- .../versions/010_add_flavors_to_migrations.py | 44 +++++++++++++++++++ nova/db/sqlalchemy/models.py | 2 + nova/tests/test_compute.py | 2 - 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/010_add_flavors_to_migrations.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_flavors_to_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/010_add_flavors_to_migrations.py new file mode 100644 index 0000000000..412caedd0e --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/010_add_flavors_to_migrations.py @@ -0,0 +1,44 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.from sqlalchemy import * + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +migrations = Table('migrations', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# Tables to alter +# +# + +old_flavor_id = Column('old_flavor_id', Integer()) +new_flavor_id = Column('new_flavor_id', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + migrations.create_column(old_flavor_id) + migrations.create_column(new_flavor_id) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 6ef284e65e..73cd8a4ccf 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -396,6 +396,8 @@ class Migration(BASE, NovaBase): source_compute = Column(String(255)) dest_compute = Column(String(255)) dest_host = Column(String(255)) + old_flavor_id = Column(Integer()) + new_flavor_id = Column(Integer()) instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) #TODO(_cerberus_): enum status = Column(String(255)) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 643b2e93a6..3d25a8997c 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -299,5 +299,3 @@ class ComputeTestCase(test.TestCase): self.assertRaises(exception.Error, self.compute.prep_resize, self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) - type = instance_types.get_by_flavor_id("1") - self.assertEqual(type, 'm1.tiny') From 1c4afe23157233b7081872ccbc6ea5fa1ff0015a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Mar 2011 17:30:51 -0600 Subject: [PATCH 004/101] Some unit tests --- nova/compute/api.py | 9 ++++++--- nova/tests/test_compute.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 1393c01d52..0dc2bb3d3f 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -460,10 +460,13 @@ class API(base.Base): def resize(self, context, instance_id, flavor_id): """Resize a running instance.""" instance = self.db.instance_get(context, instance_id) - current_instance_type = self.db.instance_type_get_by_flavor_id( - context, instance['flavor_id']) + current_instance_type = self.db.instance_type_get_by_name( + context, instance['instance_type']) + new_instance_type = self.db.instance_type_get_by_flavor_id( - context, flavor_id) + context, flavor_id) + if not new_instance_type: + raise exception.ApiError(_("Requested flavor does not exist")) if current_instance_type.memory_mb > new_instance_typ.memory_mb: raise exception.ApiError(_("Invalid flavor: cannot downsize" diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 3d25a8997c..c532842169 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -287,6 +287,30 @@ class ComputeTestCase(test.TestCase): migration_ref['id']) self.compute.terminate_instance(context, instance_id) + def test_resize_invalid_flavor_fails(self): + """Ensure invalid flavors raise""" + instance_id = self._create_instance() + context = self.context.elevated() + self.compute.run_instance(self.context, instance_id) + + self.assertRaises(exception.ApiError, self.compute_api.resize, + context, instance_id, 200) + + self.compute.terminate_instance(context, instance_id) + + def test_resize_down_fails(self): + """Ensure invalid flavors raise""" + instance_id = self._create_instance() + context = self.context.elevated() + self.compute.run_instance(self.context, instance_id) + db.instance_update(self.context, instance_id, + {'instance_type': 'm1.large'}) + + self.assertRaises(exception.ApiError, self.compute_api.resize, + context, instance_id, 1) + + self.compute.terminate_instance(context, instance_id) + def test_get_by_flavor_id(self): type = instance_types.get_by_flavor_id(1) self.assertEqual(type, 'm1.tiny') From af5e752e8eb21d0e9192d9acd9e75586bdec3685 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Mar 2011 11:55:55 -0500 Subject: [PATCH 005/101] Compute test --- nova/tests/test_compute.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index c532842169..47e0f66fb9 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -76,6 +76,20 @@ class ComputeTestCase(test.TestCase): inst.update(params) return db.instance_create(self.context, inst)['id'] + def _create_instance_type(self, params={}): + """Create a test instance""" + inst = {} + inst['name'] = 'm1.small' + inst['memory_mb'] = '1024' + inst['vcpus'] = '1' + inst['local_gb'] = '20' + inst['flavorid'] = '1' + inst['swap'] = '2048' + inst['rxtx_quota'] = 100 + inst['rxtx_cap'] = 200 + inst.update(params) + return db.instance_type_create(self.context, inst)['id'] + def _create_group(self): values = {'name': 'testgroup', 'description': 'testgroup', @@ -301,10 +315,17 @@ class ComputeTestCase(test.TestCase): def test_resize_down_fails(self): """Ensure invalid flavors raise""" instance_id = self._create_instance() + + small_inst_type_id = self._create_instance_type(dict(flavorid=1, + memory_mb=512)) + big_inst_type_id = self._create_instance_type(dict(flavorid=2, + name='m1.wowzers', memory_mb=8192)) + context = self.context.elevated() self.compute.run_instance(self.context, instance_id) - db.instance_update(self.context, instance_id, - {'instance_type': 'm1.large'}) + db.instance_update(self.context, instance_id, + {'instance_type': 'm1.wowzers', + 'memory_gb': 8192}) self.assertRaises(exception.ApiError, self.compute_api.resize, context, instance_id, 1) From 1f763599d733de1ded1074dee828237256eda01d Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Mon, 14 Mar 2011 16:59:46 +0000 Subject: [PATCH 006/101] Migration moved again --- ..._flavors_to_migrations.py => 011_add_flavors_to_migrations.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nova/db/sqlalchemy/migrate_repo/versions/{010_add_flavors_to_migrations.py => 011_add_flavors_to_migrations.py} (100%) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_flavors_to_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/011_add_flavors_to_migrations.py similarity index 100% rename from nova/db/sqlalchemy/migrate_repo/versions/010_add_flavors_to_migrations.py rename to nova/db/sqlalchemy/migrate_repo/versions/011_add_flavors_to_migrations.py From 1ebae577150ce64d81d102c2e162acfe5a72528b Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Mar 2011 12:07:27 -0500 Subject: [PATCH 007/101] Test changes --- nova/compute/api.py | 2 +- nova/tests/test_compute.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 3920f2af85..9d238c7d09 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -482,7 +482,7 @@ class API(base.Base): {"method": "prep_resize", "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, - "instance_type": new_instance_type}}) + "flavor_id": flavor_id}}) def pause(self, context, instance_id): """Pause the given instance.""" diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 47e0f66fb9..265421837f 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -292,14 +292,18 @@ class ComputeTestCase(test.TestCase): """Ensure instance can be migrated/resized""" instance_id = self._create_instance() context = self.context.elevated() + small_inst_type_id = self._create_instance_type(dict(flavorid=1, + memory_mb=512, name='m1.small')) + self.compute.run_instance(self.context, instance_id) db.instance_update(self.context, instance_id, {'host': 'foo'}) - self.compute.prep_resize(context, instance_id) + self.compute.prep_resize(context, instance_id, 1) migration_ref = db.migration_get_by_instance_and_status(context, instance_id, 'pre-migrating') self.compute.resize_instance(context, instance_id, migration_ref['id']) self.compute.terminate_instance(context, instance_id) + self.db.instance_type_purge(context, 'm1.small') def test_resize_invalid_flavor_fails(self): """Ensure invalid flavors raise""" @@ -317,7 +321,7 @@ class ComputeTestCase(test.TestCase): instance_id = self._create_instance() small_inst_type_id = self._create_instance_type(dict(flavorid=1, - memory_mb=512)) + memory_mb=512, name='m1.small')) big_inst_type_id = self._create_instance_type(dict(flavorid=2, name='m1.wowzers', memory_mb=8192)) @@ -331,6 +335,8 @@ class ComputeTestCase(test.TestCase): context, instance_id, 1) self.compute.terminate_instance(context, instance_id) + self.db.instance_type_purge(context, 'm1.small') + self.db.instance_type_purge(context, 'm1.wowzers') def test_get_by_flavor_id(self): type = instance_types.get_by_flavor_id(1) @@ -340,7 +346,10 @@ class ComputeTestCase(test.TestCase): """Ensure instance fails to migrate when source and destination are the same host""" instance_id = self._create_instance() + small_inst_type_id = self._create_instance_type(dict(flavorid=1, + memory_mb=512, name='m1.small')) self.compute.run_instance(self.context, instance_id) self.assertRaises(exception.Error, self.compute.prep_resize, - self.context, instance_id) + self.context, instance_id, 1) self.compute.terminate_instance(self.context, instance_id) + self.db.instance_type_purge(context, 'm1.small') From e509cd70e7a2e8a430b2b24af50adcf1ad763564 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Mon, 14 Mar 2011 17:24:39 +0000 Subject: [PATCH 008/101] Test fixes and some typos --- nova/compute/api.py | 4 ++-- nova/compute/manager.py | 2 +- nova/db/sqlalchemy/api.py | 4 ++-- nova/tests/test_compute.py | 23 +++++------------------ 4 files changed, 10 insertions(+), 23 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 9d238c7d09..0e9bf2424a 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -468,13 +468,13 @@ class API(base.Base): instance = self.db.instance_get(context, instance_id) current_instance_type = self.db.instance_type_get_by_name( context, instance['instance_type']) - + new_instance_type = self.db.instance_type_get_by_flavor_id( context, flavor_id) if not new_instance_type: raise exception.ApiError(_("Requested flavor does not exist")) - if current_instance_type.memory_mb > new_instance_typ.memory_mb: + if current_instance_type['memory_mb'] > new_instance_type['memory_mb']: raise exception.ApiError(_("Invalid flavor: cannot downsize" "instances")) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index f73e813452..57d175ec7b 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -479,7 +479,7 @@ class ComputeManager(manager.Manager): 'source_compute': instance_ref['host'], 'dest_compute': FLAGS.host, 'dest_host': self.driver.get_host_ip_addr(), - 'old_flavor_id': instance_type['flavor_id'], + 'old_flavor_id': instance_type['flavorid'], 'new_flavor_id': flavor_id, 'status': 'pre-migrating'}) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 8b541757aa..50267e21fd 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2185,8 +2185,8 @@ def instance_type_create(_context, values): instance_type_ref = models.InstanceTypes() instance_type_ref.update(values) instance_type_ref.save() - except: - raise exception.DBError + except Exception, e: + raise exception.DBError(e) return instance_type_ref diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 265421837f..a6defd644f 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -78,6 +78,7 @@ class ComputeTestCase(test.TestCase): def _create_instance_type(self, params={}): """Create a test instance""" + context = self.context.elevated() inst = {} inst['name'] = 'm1.small' inst['memory_mb'] = '1024' @@ -88,7 +89,7 @@ class ComputeTestCase(test.TestCase): inst['rxtx_quota'] = 100 inst['rxtx_cap'] = 200 inst.update(params) - return db.instance_type_create(self.context, inst)['id'] + return db.instance_type_create(context, inst)['id'] def _create_group(self): values = {'name': 'testgroup', @@ -292,8 +293,6 @@ class ComputeTestCase(test.TestCase): """Ensure instance can be migrated/resized""" instance_id = self._create_instance() context = self.context.elevated() - small_inst_type_id = self._create_instance_type(dict(flavorid=1, - memory_mb=512, name='m1.small')) self.compute.run_instance(self.context, instance_id) db.instance_update(self.context, instance_id, {'host': 'foo'}) @@ -303,7 +302,6 @@ class ComputeTestCase(test.TestCase): self.compute.resize_instance(context, instance_id, migration_ref['id']) self.compute.terminate_instance(context, instance_id) - self.db.instance_type_purge(context, 'm1.small') def test_resize_invalid_flavor_fails(self): """Ensure invalid flavors raise""" @@ -311,32 +309,24 @@ class ComputeTestCase(test.TestCase): context = self.context.elevated() self.compute.run_instance(self.context, instance_id) - self.assertRaises(exception.ApiError, self.compute_api.resize, + self.assertRaises(exception.NotFound, self.compute_api.resize, context, instance_id, 200) self.compute.terminate_instance(context, instance_id) def test_resize_down_fails(self): """Ensure invalid flavors raise""" + context = self.context.elevated() instance_id = self._create_instance() - small_inst_type_id = self._create_instance_type(dict(flavorid=1, - memory_mb=512, name='m1.small')) - big_inst_type_id = self._create_instance_type(dict(flavorid=2, - name='m1.wowzers', memory_mb=8192)) - - context = self.context.elevated() self.compute.run_instance(self.context, instance_id) db.instance_update(self.context, instance_id, - {'instance_type': 'm1.wowzers', - 'memory_gb': 8192}) + {'instance_type': 'm1.xlarge'}) self.assertRaises(exception.ApiError, self.compute_api.resize, context, instance_id, 1) self.compute.terminate_instance(context, instance_id) - self.db.instance_type_purge(context, 'm1.small') - self.db.instance_type_purge(context, 'm1.wowzers') def test_get_by_flavor_id(self): type = instance_types.get_by_flavor_id(1) @@ -346,10 +336,7 @@ class ComputeTestCase(test.TestCase): """Ensure instance fails to migrate when source and destination are the same host""" instance_id = self._create_instance() - small_inst_type_id = self._create_instance_type(dict(flavorid=1, - memory_mb=512, name='m1.small')) self.compute.run_instance(self.context, instance_id) self.assertRaises(exception.Error, self.compute.prep_resize, self.context, instance_id, 1) self.compute.terminate_instance(self.context, instance_id) - self.db.instance_type_purge(context, 'm1.small') From 67c871a257c684de3cb0f1416b1b2b6e9a99fe23 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Mar 2011 17:37:07 -0500 Subject: [PATCH 009/101] Moving the migration again --- ..._flavors_to_migrations.py => 012_add_flavors_to_migrations.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nova/db/sqlalchemy/migrate_repo/versions/{011_add_flavors_to_migrations.py => 012_add_flavors_to_migrations.py} (100%) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/011_add_flavors_to_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py similarity index 100% rename from nova/db/sqlalchemy/migrate_repo/versions/011_add_flavors_to_migrations.py rename to nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py From 74987666f89b4d15ffcf17b43b3752135ba08a65 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Mar 2011 18:48:17 -0500 Subject: [PATCH 010/101] A few fixes --- nova/compute/manager.py | 2 +- nova/virt/xenapi/vmops.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 39b28f6a94..307c916501 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -541,7 +541,7 @@ class ComputeManager(manager.Manager): #after they're supported instance_type = self.db.instance_type_get_by_flavor_id(context, migration_ref['new_flavor_id']) - self.db.instance_update(context, instance_ref, + self.db.instance_update(context, instance_id, dict(memory_mb=instance_type['memory_mb'], vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d1aaf998f5..119d6dba85 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -304,7 +304,7 @@ class VMOps(object): try: # transfer the base copy template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) - base_copy_uuid = template_vdi_uuids[1] + base_copy_uuid = template_vdi_uuids['snap'] vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) cow_uuid = vm_vdi_rec['uuid'] @@ -319,7 +319,7 @@ class VMOps(object): self._session.wait_for_task(task, instance.id) # Now power down the instance and transfer the COW VHD - self._shutdown(instance, vm_ref, method='clean') + self._shutdown(instance, vm_ref) params = {'host': dest, 'vdi_uuid': cow_uuid, From 39e722b58b87297aee770637f6a82ee1f206aecf Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Mar 2011 18:51:22 -0500 Subject: [PATCH 011/101] Tweak --- nova/virt/xenapi/vmops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 119d6dba85..958201695c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -319,7 +319,7 @@ class VMOps(object): self._session.wait_for_task(task, instance.id) # Now power down the instance and transfer the COW VHD - self._shutdown(instance, vm_ref) + self._shutdown(instance, vm_ref, hard=False) params = {'host': dest, 'vdi_uuid': cow_uuid, @@ -447,7 +447,8 @@ class VMOps(object): """Shutdown an instance""" state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: - LOG.warn(_("VM %(vm)s already halted, skipping shutdown...") % + instance_name = instance.name + LOG.warn(_("VM %(instance_name)s already halted, skipping shutdown...") % locals()) return From 9650e73db3e18f839f8abf7a47aebb6fbf8c9e36 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Mar 2011 19:10:50 -0500 Subject: [PATCH 012/101] Plugin --- nova/virt/xenapi/vmops.py | 1 - plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 958201695c..cdc4a417c9 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -370,7 +370,6 @@ class VMOps(object): #TODO(mdietz): this will need to be adjusted for swap later task = self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) - vm_ref = VMHelper.lookup(self._session, instance.name) self._session.wait_for_task(task, instance.id) def reboot(self, instance): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 4aa89863a3..6008e71bf1 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -22,6 +22,7 @@ XenAPI Plugin for transfering data between host nodes import os import os.path import pickle +import shlex import shutil import subprocess From d1469d1566a67d41cb4de4ff06deaf441e099062 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 11:26:40 -0500 Subject: [PATCH 013/101] Some typos --- nova/compute/api.py | 4 +++- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 0e9bf2424a..08947eb3af 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -443,6 +443,8 @@ class API(base.Base): params = {'migration_id': migration_ref['id']} self._cast_compute_message('revert_resize', context, instance_id, migration_ref['dest_compute'], params=params) + self.db.migration_update(context, migration_ref['id'], + {'status': 'reverted'}) def confirm_resize(self, context, instance_id): """Confirms a migration/resize, deleting the 'old' instance in the @@ -458,7 +460,7 @@ class API(base.Base): self._cast_compute_message('confirm_resize', context, instance_id, migration_ref['source_compute'], params=params) - self.db.migration_update(context, migration_id, + self.db.migration_update(context, migration_ref['id'], {'status': 'confirmed'}) self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 6008e71bf1..75c6534087 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -98,7 +98,7 @@ def transfer_vhd(session, args): logging.debug("Preparing to transmit %s to %s" % (source_path, dest_path)) - ssh_cmd = 'ssh -o StrictHostKeyChecking=no' + ssh_cmd = '\"ssh -o StrictHostKeyChecking=no\"' rsync_args = shlex.split('nohup /usr/bin/rsync -av --progress -e %s %s %s' % (ssh_cmd, source_path, dest_path)) From 9cb503ae9d4112fa464f2284631ad1e24f8f7ce4 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 11:38:40 -0500 Subject: [PATCH 014/101] Stuff --- nova/virt/xenapi/vmops.py | 3 +-- nova/virt/xenapi_conn.py | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index cdc4a417c9..ebaa4a69a5 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -363,8 +363,7 @@ class VMOps(object): def resize_instance(self, instance, vdi_uuid): """Resize a running instance by changing it's RAM and disk size """ vm_ref = VMHelper.lookup(self._session, instance.name) - vdi_ref, vm_vdi_rec = \ - VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) new_disk_size = instance.local_gb #TODO(mdietz): this will need to be adjusted for swap later diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 6b1b51fee7..b8256d205d 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -164,15 +164,11 @@ class XenAPIConnection(object): """Create VM instance""" self._vmops.spawn(instance) - def resize_instance(self, instance, disk_info): - """Resizes instance attributes such as RAM and disk space to the - attributes specified by the record""" - self._vmops.resize_instance(instance, disk_info['cow']) - def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" vdi_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], disk_info['cow']) + self._vmops.resize_instance(instance, vdi_uuid) self._vmops._spawn_with_disk(instance, vdi_uuid) def snapshot(self, instance, image_id): From dee86f53b0d1dccbc69d354b66ca7a4767e81d43 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 11:54:10 -0500 Subject: [PATCH 015/101] tweak --- nova/compute/manager.py | 1 - nova/virt/xenapi/vmops.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 307c916501..1587660a37 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -546,7 +546,6 @@ class ComputeManager(manager.Manager): vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) - self.driver.resize_instance(instance_ref, disk_info) self.driver.finish_resize(instance_ref, disk_info) self.db.migration_update(context, migration_id, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ebaa4a69a5..483b0cb823 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -364,7 +364,7 @@ class VMOps(object): """Resize a running instance by changing it's RAM and disk size """ vm_ref = VMHelper.lookup(self._session, instance.name) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - new_disk_size = instance.local_gb + new_disk_size = instance.local_gb * 1024 #TODO(mdietz): this will need to be adjusted for swap later task = self._session.call_xenapi('VDI.resize_online', vdi_ref, From d99a8d48cf38eb6be01587f9b377f48ff6cd88a2 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 16 Mar 2011 17:09:13 +0000 Subject: [PATCH 016/101] Logging statements --- nova/virt/xenapi/vmops.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 483b0cb823..c292822cac 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -363,13 +363,16 @@ class VMOps(object): def resize_instance(self, instance, vdi_uuid): """Resize a running instance by changing it's RAM and disk size """ vm_ref = VMHelper.lookup(self._session, instance.name) - vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) new_disk_size = instance.local_gb * 1024 + LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %d megs") % (vdi_uuid, + instance.name, new_disk_size)) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) #TODO(mdietz): this will need to be adjusted for swap later task = self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) self._session.wait_for_task(task, instance.id) + LOG.debug(_("Resize instance %s complete") % (instance.name)) def reboot(self, instance): """Reboot VM instance""" From a31e715617e5af107bc79caeedf0aff41f65fb07 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 12:57:45 -0500 Subject: [PATCH 017/101] The geebees --- nova/virt/xenapi/vmops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c292822cac..b449437c97 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -363,7 +363,7 @@ class VMOps(object): def resize_instance(self, instance, vdi_uuid): """Resize a running instance by changing it's RAM and disk size """ vm_ref = VMHelper.lookup(self._session, instance.name) - new_disk_size = instance.local_gb * 1024 + new_disk_size = str(instance.local_gb * 1024 * 1024) LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %d megs") % (vdi_uuid, instance.name, new_disk_size)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) From e2399c434386a31114273f2cf6f14586a25480c2 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 13:06:49 -0500 Subject: [PATCH 018/101] Derped again --- nova/virt/xenapi/vmops.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b449437c97..7f80de8a92 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -363,8 +363,10 @@ class VMOps(object): def resize_instance(self, instance, vdi_uuid): """Resize a running instance by changing it's RAM and disk size """ vm_ref = VMHelper.lookup(self._session, instance.name) - new_disk_size = str(instance.local_gb * 1024 * 1024) - LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %d megs") % (vdi_uuid, + + #The new disk size must be in bytes + new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) + LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %s megs") % (vdi_uuid, instance.name, new_disk_size)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) From 647f5f0d0283b3852115d821b80a965b0bc92c35 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 13:24:51 -0500 Subject: [PATCH 019/101] chchchchchanges --- nova/virt/xenapi/vmops.py | 19 ++++++++++--------- nova/virt/xenapi_conn.py | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 7f80de8a92..6ff0aad15c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -72,7 +72,7 @@ class VMOps(object): LOG.debug(_("Starting instance %s"), instance.name) self._session.call_xenapi('VM.start', vm_ref, False, False) - def create_disk(self, instance): + def _create_disk(self, instance): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) disk_image_type = VMHelper.determine_disk_image_type(instance) @@ -81,11 +81,11 @@ class VMOps(object): return vdi_uuid def spawn(self, instance): - vdi_uuid = self.create_disk(instance) - self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) + vdi_uuid = self._create_disk(instance) + vm_ref = self._create_vm(instance, vdi_uuid) + self._spawn(instance, vm_ref) - def _spawn_with_disk(self, instance, vdi_uuid): - """Create VM instance""" + def _create_vm(self, instance, vdi_uuid): instance_name = instance.name vm_ref = VMHelper.lookup(self._session, instance_name) if vm_ref is not None: @@ -130,7 +130,10 @@ class VMOps(object): # inject_network_info and create vifs networks = self.inject_network_info(instance) self.create_vifs(instance, networks) + return vm_ref + def _spawn(self, instance, vm_ref): + """Spawn a new instance""" LOG.debug(_('Starting VM %s...'), vm_ref) self._start(instance, vm_ref) LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') @@ -364,16 +367,14 @@ class VMOps(object): """Resize a running instance by changing it's RAM and disk size """ vm_ref = VMHelper.lookup(self._session, instance.name) + #TODO(mdietz): this will need to be adjusted for swap later #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %s megs") % (vdi_uuid, instance.name, new_disk_size)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - #TODO(mdietz): this will need to be adjusted for swap later - task = self._session.call_xenapi('VDI.resize_online', vdi_ref, - new_disk_size) - self._session.wait_for_task(task, instance.id) + self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) LOG.debug(_("Resize instance %s complete") % (instance.name)) def reboot(self, instance): diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index b8256d205d..fd68c0fe77 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -168,8 +168,9 @@ class XenAPIConnection(object): """Completes a resize, turning on the migrated instance""" vdi_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], disk_info['cow']) + self._vmops._create_vm(instance, vdi_uuid) self._vmops.resize_instance(instance, vdi_uuid) - self._vmops._spawn_with_disk(instance, vdi_uuid) + self._vmops._spawn_with_disk(instance) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ From 11e7b6a08d1557a0986b480c032958cd30762f33 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 13:31:05 -0500 Subject: [PATCH 020/101] chchchchchanges --- nova/virt/xenapi/vmops.py | 2 -- nova/virt/xenapi_conn.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6ff0aad15c..92594c9c6f 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -365,8 +365,6 @@ class VMOps(object): def resize_instance(self, instance, vdi_uuid): """Resize a running instance by changing it's RAM and disk size """ - vm_ref = VMHelper.lookup(self._session, instance.name) - #TODO(mdietz): this will need to be adjusted for swap later #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index fd68c0fe77..046f74c8dd 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -168,9 +168,9 @@ class XenAPIConnection(object): """Completes a resize, turning on the migrated instance""" vdi_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], disk_info['cow']) - self._vmops._create_vm(instance, vdi_uuid) + vm_ref = self._vmops._create_vm(instance, vdi_uuid) self._vmops.resize_instance(instance, vdi_uuid) - self._vmops._spawn_with_disk(instance) + self._vmops._spawn(instance, vm_ref) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ From d8c3ea5e6b594e6285650c5bdac6302b7be295dc Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 13:39:43 -0500 Subject: [PATCH 021/101] chchchchchanges --- nova/virt/xenapi/vmops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 92594c9c6f..931fc1cb41 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -136,6 +136,7 @@ class VMOps(object): """Spawn a new instance""" LOG.debug(_('Starting VM %s...'), vm_ref) self._start(instance, vm_ref) + instance_name = instance.name LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) From ebd452eab95c2f205d3f7419c08c288030c38aba Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 13:53:49 -0500 Subject: [PATCH 022/101] chchchchchanges --- nova/compute/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 1587660a37..351e02f515 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -546,6 +546,7 @@ class ComputeManager(manager.Manager): vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) + instance_ref = self.db.instance_get(context, instance_id) self.driver.finish_resize(instance_ref, disk_info) self.db.migration_update(context, migration_id, From 1d4d0e26ae6ece5e68417deaa4ddcf4b7757bd37 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 14:09:14 -0500 Subject: [PATCH 023/101] Fudge --- nova/compute/api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index 08947eb3af..ddf439b356 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -468,11 +468,15 @@ class API(base.Base): def resize(self, context, instance_id, flavor_id): """Resize a running instance.""" instance = self.db.instance_get(context, instance_id) + LOG.debug(_("Resizing instance %s to flavor %d") % + (instance.name, flavor_id)) current_instance_type = self.db.instance_type_get_by_name( context, instance['instance_type']) new_instance_type = self.db.instance_type_get_by_flavor_id( context, flavor_id) + LOG.debug(_("Old instance type %s -> New instance type %s") % + (current_instance_type['name'], new_instance_type['name'])) if not new_instance_type: raise exception.ApiError(_("Requested flavor does not exist")) From 3459cfb89bd90605e54fd1fb28b8b38089f3e236 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 16 Mar 2011 15:20:08 -0400 Subject: [PATCH 024/101] update image service documentation --- nova/image/service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/image/service.py b/nova/image/service.py index c09052cab6..78d8f33e97 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -40,9 +40,9 @@ class BaseImageService(object): :retval: a sequence of mappings with the following signature {'id': opaque id of image, 'name': name of image, - 'created_at': creation timestamp, - 'updated_at': modification timestamp, - 'deleted_at': deletion timestamp or None, + 'created_at': creation datetime object, + 'updated_at': modification datetime object, + 'deleted_at': deletion datetime object or None, 'deleted': boolean indicating if image has been deleted, 'status': string description of image status, 'is_public': boolean indicating if image is public @@ -64,9 +64,9 @@ class BaseImageService(object): {'id': opaque id of image, 'name': name of image, - 'created_at': creation timestamp, - 'updated_at': modification timestamp, - 'deleted_at': deletion timestamp or None, + 'created_at': creation datetime object, + 'updated_at': modification datetime object, + 'deleted_at': deletion datetime object or None, 'deleted': boolean indicating if image has been deleted, 'status': string description of image status, 'is_public': boolean indicating if image is public From 007c2802e542bf954f0aa5b589f2adc3a1bfa89a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 15:41:53 -0500 Subject: [PATCH 025/101] Reverting --- nova/virt/xenapi/vmops.py | 10 +++------- nova/virt/xenapi_conn.py | 8 ++++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 931fc1cb41..7525ff5ec4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -85,7 +85,8 @@ class VMOps(object): vm_ref = self._create_vm(instance, vdi_uuid) self._spawn(instance, vm_ref) - def _create_vm(self, instance, vdi_uuid): + def _spawn(self, instance, vdi_uuid): + """Spawn a new instance""" instance_name = instance.name vm_ref = VMHelper.lookup(self._session, instance_name) if vm_ref is not None: @@ -130,13 +131,8 @@ class VMOps(object): # inject_network_info and create vifs networks = self.inject_network_info(instance) self.create_vifs(instance, networks) - return vm_ref - - def _spawn(self, instance, vm_ref): - """Spawn a new instance""" LOG.debug(_('Starting VM %s...'), vm_ref) self._start(instance, vm_ref) - instance_name = instance.name LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) @@ -343,7 +339,7 @@ class VMOps(object): # sensible so we don't need to blindly pass around dictionaries return {'base_copy': base_copy_uuid, 'cow': cow_uuid} - def attach_disk(self, instance, base_copy_uuid, cow_uuid): + def link_disks(self, instance, base_copy_uuid, cow_uuid): """Links the base copy VHD to the COW via the XAPI plugin""" vm_ref = VMHelper.lookup(self._session, instance.name) new_base_copy_uuid = str(uuid.uuid4()) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 046f74c8dd..99ec53c118 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -166,11 +166,11 @@ class XenAPIConnection(object): def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" - vdi_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], + vdi_uuid = self._vmops.link_disks(instance, disk_info['base_copy'], disk_info['cow']) - vm_ref = self._vmops._create_vm(instance, vdi_uuid) - self._vmops.resize_instance(instance, vdi_uuid) - self._vmops._spawn(instance, vm_ref) + #vm_ref = self._vmops._create_vm(instance, vdi_uuid) + #self._vmops.resize_instance(instance, vdi_uuid) + self._vmops._spawn(instance, vdi_uuid) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ From 0e63a45f40a2069d497878b7c05d00522c3a2774 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 16:24:38 -0500 Subject: [PATCH 026/101] Again --- nova/virt/xenapi/vmops.py | 13 ++++++++----- nova/virt/xenapi_conn.py | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 7525ff5ec4..ab98ef000a 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -85,8 +85,7 @@ class VMOps(object): vm_ref = self._create_vm(instance, vdi_uuid) self._spawn(instance, vm_ref) - def _spawn(self, instance, vdi_uuid): - """Spawn a new instance""" + def _create_vm(self, instance, vdi_uuid): instance_name = instance.name vm_ref = VMHelper.lookup(self._session, instance_name) if vm_ref is not None: @@ -131,8 +130,13 @@ class VMOps(object): # inject_network_info and create vifs networks = self.inject_network_info(instance) self.create_vifs(instance, networks) + return vm_ref + + def _spawn(self, instance, vm_ref): + """Spawn a new instance""" LOG.debug(_('Starting VM %s...'), vm_ref) self._start(instance, vm_ref) + instance_name = instance.name LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) @@ -365,10 +369,9 @@ class VMOps(object): #TODO(mdietz): this will need to be adjusted for swap later #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %s megs") % (vdi_uuid, - instance.name, new_disk_size)) + LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB") % (vdi_uuid, + instance.name, instance.local_gb)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) LOG.debug(_("Resize instance %s complete") % (instance.name)) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 99ec53c118..2b0f82a4a1 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -168,9 +168,9 @@ class XenAPIConnection(object): """Completes a resize, turning on the migrated instance""" vdi_uuid = self._vmops.link_disks(instance, disk_info['base_copy'], disk_info['cow']) - #vm_ref = self._vmops._create_vm(instance, vdi_uuid) - #self._vmops.resize_instance(instance, vdi_uuid) - self._vmops._spawn(instance, vdi_uuid) + vm_ref = self._vmops._create_vm(instance, vdi_uuid) + self._vmops.resize_instance(instance, vdi_uuid) + self._vmops._spawn(instance, vm_ref) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ From c7da5632e954c860defc322e971936a8d60eb8fd Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 16:55:58 -0500 Subject: [PATCH 027/101] foo --- nova/virt/xenapi/vmops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ab98ef000a..9719e05b99 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -308,7 +308,7 @@ class VMOps(object): try: # transfer the base copy template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) - base_copy_uuid = template_vdi_uuids['snap'] + base_copy_uuid = template_vdi_uuids['image'] vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) cow_uuid = vm_vdi_rec['uuid'] From cc2d4728d32d016ef803d0def456cac6e315e8fa Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 16 Mar 2011 17:56:40 -0400 Subject: [PATCH 028/101] get started testing --- nova/image/glance.py | 6 ++++-- nova/tests/image/__init__.py | 0 nova/tests/image/test_glance.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 nova/tests/image/__init__.py create mode 100644 nova/tests/image/test_glance.py diff --git a/nova/image/glance.py b/nova/image/glance.py index 15fca69b8b..3b448db4b5 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -37,8 +37,10 @@ GlanceClient = utils.import_class('glance.client.Client') class GlanceImageService(service.BaseImageService): """Provides storage and retrieval of disk image objects within Glance.""" - def __init__(self): - self.client = GlanceClient(FLAGS.glance_host, FLAGS.glance_port) + def __init__(self, client=None): + if client is None: + self.client = GlanceClient(FLAGS.glance_host, FLAGS.glance_port) + self.client = client def index(self, context): """ diff --git a/nova/tests/image/__init__.py b/nova/tests/image/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py new file mode 100644 index 0000000000..b568f593de --- /dev/null +++ b/nova/tests/image/test_glance.py @@ -0,0 +1,18 @@ +import unittest + +from nova.image import glance + +class StubGlanceClient(object): + + def __init__(self, images): + self._images = images + + def get_image_meta(id): + return self._images[id] + +class TestGlance(unittest.TestCase): + + def test(self): + images = {'xyz': "image"} + client = StubGlanceClient(images) + service = glance.GlanceImageService(client) From 8385599f941c5fe886de570b67f5e57e64e96468 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 16:58:46 -0500 Subject: [PATCH 029/101] hurr --- nova/db/sqlalchemy/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index f4773ce324..84db330ecb 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2206,8 +2206,8 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(instance_id=instance_id).\ filter_by(status=status).first() if not result: - raise exception.NotFound(_("No migration found with instance id %s") - % migration_id) + raise exception.NotFound(_("No migration found for instance %d") + "with status %s" % (instance_id, status)) return result From 524eb966045192dd535648929d70cac091d8e24e Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 17:00:22 -0500 Subject: [PATCH 030/101] hurr --- nova/db/sqlalchemy/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 84db330ecb..7e358e64b8 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2206,8 +2206,8 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(instance_id=instance_id).\ filter_by(status=status).first() if not result: - raise exception.NotFound(_("No migration found for instance %d") - "with status %s" % (instance_id, status)) + raise exception.NotFound(_("No migration found for instance %d" + "with status %s" % (instance_id, status))) return result From bf2f491f3e7aa5522d306c2182c3d220eb49a55f Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 17:56:48 -0500 Subject: [PATCH 031/101] foo --- nova/compute/manager.py | 5 +++-- nova/db/sqlalchemy/api.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 351e02f515..e69544b6e4 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -458,7 +458,7 @@ class ComputeManager(manager.Manager): #Just roll back the record. There's no need to resize down since #the 'old' VM already has the preferred attributes - self.db.instance_update(context, + self.db.instance_update(context, instance_id dict(memory_mb=instance_type['memory_mb'], vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) @@ -542,7 +542,8 @@ class ComputeManager(manager.Manager): instance_type = self.db.instance_type_get_by_flavor_id(context, migration_ref['new_flavor_id']) self.db.instance_update(context, instance_id, - dict(memory_mb=instance_type['memory_mb'], + dict(instance_type=instance_type['name'], + memory_mb=instance_type['memory_mb'], vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 7e358e64b8..47b84af509 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2206,7 +2206,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(instance_id=instance_id).\ filter_by(status=status).first() if not result: - raise exception.NotFound(_("No migration found for instance %d" + raise exception.NotFound(_("No migration found for instance %s" "with status %s" % (instance_id, status))) return result From 3c0ae08b71c860383c215fa30c36693fd80f34c2 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 17:58:16 -0500 Subject: [PATCH 032/101] foo --- nova/compute/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index e69544b6e4..3135d58019 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -458,7 +458,7 @@ class ComputeManager(manager.Manager): #Just roll back the record. There's no need to resize down since #the 'old' VM already has the preferred attributes - self.db.instance_update(context, instance_id + self.db.instance_update(context, instance_id, dict(memory_mb=instance_type['memory_mb'], vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) From 11698a131fe6b99bfd91a977a975b07bcd4c2b2b Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Thu, 17 Mar 2011 03:21:09 -0400 Subject: [PATCH 033/101] Added mechanism for versioned controllers for openstack api versions 1.0/1.1. Create servers in the 1.1 api now supports imageRef/flavorRef instead of imageId/flavorId. --- etc/api-paste.ini | 18 ++++-- nova/api/openstack/__init__.py | 29 ++++++++-- nova/api/openstack/auth.py | 2 +- nova/api/openstack/common.py | 1 + nova/api/openstack/servers.py | 64 +++++++++++++++++---- nova/api/openstack/views/addresses.py | 16 +----- nova/api/openstack/views/flavors.py | 18 +----- nova/api/openstack/views/images.py | 18 +----- nova/api/openstack/views/servers.py | 49 ++++++---------- nova/tests/api/openstack/fakes.py | 18 +++--- nova/tests/api/openstack/test_auth.py | 6 +- nova/tests/api/openstack/test_servers.py | 73 +++++++++++++++++++++--- 12 files changed, 191 insertions(+), 121 deletions(-) diff --git a/etc/api-paste.ini b/etc/api-paste.ini index a4483d3f8a..2b395aa0c5 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -67,11 +67,14 @@ paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.f [composite:osapi] use = egg:Paste#urlmap /: osversions -/v1.0: openstackapi -/v1.1: openstackapi +/v1.0: openstackapi10 +/v1.1: openstackapi11 -[pipeline:openstackapi] -pipeline = faultwrap auth ratelimit osapiapp +[pipeline:openstackapi10] +pipeline = faultwrap auth ratelimit osapiapp10 + +[pipeline:openstackapi11] +pipeline = faultwrap auth ratelimit osapiapp11 [filter:faultwrap] paste.filter_factory = nova.api.openstack:FaultWrapper.factory @@ -82,8 +85,11 @@ paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory [filter:ratelimit] paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory -[app:osapiapp] -paste.app_factory = nova.api.openstack:APIRouter.factory +[app:osapiapp10] +paste.app_factory = nova.api.openstack:APIRouterV10.factory + +[app:osapiapp11] +paste.app_factory = nova.api.openstack:APIRouterV11.factory [pipeline:osversions] pipeline = faultwrap osversionapp diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 0244bc93c5..0b50d17d08 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -71,9 +71,14 @@ class APIRouter(wsgi.Router): return cls() def __init__(self): + self.server_members = {} mapper = routes.Mapper() + self._setup_routes(mapper) + super(APIRouter, self).__init__(mapper) - server_members = {'action': 'POST'} + def _setup_routes(self, mapper): + server_members = self.server_members + server_members['action'] = 'POST' if FLAGS.allow_admin_api: LOG.debug(_("Including admin operations in API.")) @@ -98,10 +103,6 @@ class APIRouter(wsgi.Router): controller=accounts.Controller(), collection={'detail': 'GET'}) - mapper.resource("server", "servers", controller=servers.Controller(), - collection={'detail': 'GET'}, - member=server_members) - mapper.resource("backup_schedule", "backup_schedule", controller=backup_schedules.Controller(), parent_resource=dict(member_name='server', @@ -120,7 +121,23 @@ class APIRouter(wsgi.Router): collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) - super(APIRouter, self).__init__(mapper) + +class APIRouterV10(APIRouter): + def _setup_routes(self, mapper): + APIRouter._setup_routes(self, mapper) + mapper.resource("server", "servers", + controller=servers.ControllerV10(), + collection={'detail': 'GET'}, + member=self.server_members) + + +class APIRouterV11(APIRouter): + def _setup_routes(self, mapper): + APIRouter._setup_routes(self, mapper) + mapper.resource("server", "servers", + controller=servers.ControllerV11(), + collection={'detail': 'GET'}, + member=self.server_members) class Versions(wsgi.Application): diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 7ae285019c..6f1cf5e63f 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -70,7 +70,7 @@ class AuthMiddleware(wsgi.Middleware): req.environ['nova.context'] = context.RequestContext(user, account) version = req.path.split('/')[1].replace('v', '') - req.environ['nova.api.openstack.version'] = version + req.environ['api.version'] = version return self.application def has_authentication(self, req): diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index d94969ff57..d6679de010 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -75,5 +75,6 @@ def get_image_id_from_image_hash(image_service, context, image_hash): return image_id raise exception.NotFound(image_hash) + def get_api_version(req): return req.environ.get('api.version') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index dc62882ebb..9ce0caa463 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -27,8 +27,9 @@ from nova import wsgi from nova import utils from nova.api.openstack import common from nova.api.openstack import faults -from nova.api.openstack.views import servers as servers_views -from nova.api.openstack.views import addresses as addresses_views +import nova.api.openstack.views.addresses +import nova.api.openstack.views.flavors +import nova.api.openstack.views.servers from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state @@ -57,7 +58,7 @@ class Controller(wsgi.Controller): def ips(self, req, id): try: instance = self.compute_api.get(req.environ['nova.context'], id) - builder = addresses_views.get_view_builder(req) + builder = self._get_addresses_view_builder(req) return builder.build(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -77,7 +78,7 @@ class Controller(wsgi.Controller): """ instance_list = self.compute_api.get_all(req.environ['nova.context']) limited_list = common.limited(instance_list, req) - builder = servers_views.get_view_builder(req) + builder = self._get_view_builder(req) servers = [builder.build(inst, is_detail)['server'] for inst in limited_list] return dict(servers=servers) @@ -86,7 +87,7 @@ class Controller(wsgi.Controller): """ Returns server details by server id """ try: instance = self.compute_api.get(req.environ['nova.context'], id) - builder = servers_views.get_view_builder(req) + builder = self._get_view_builder(req) return builder.build(instance, is_detail=True) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -111,8 +112,9 @@ class Controller(wsgi.Controller): raise exception.NotFound(_("No keypairs defined")) key_pair = key_pairs[0] + requested_image_id = self._image_id_from_req_data(env) image_id = common.get_image_id_from_image_hash(self._image_service, - context, env['server']['imageId']) + context, requested_image_id) kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( req, image_id) @@ -126,9 +128,10 @@ class Controller(wsgi.Controller): for k, v in env['server']['metadata'].items(): metadata.append({'key': k, 'value': v}) - instances = self.compute_api.create( + flavor_id = self._flavor_id_from_req_data(env) + (inst,) = self.compute_api.create( context, - instance_types.get_by_flavor_id(env['server']['flavorId']), + instance_types.get_by_flavor_id(flavor_id), image_id, kernel_id=kernel_id, ramdisk_id=ramdisk_id, @@ -138,9 +141,11 @@ class Controller(wsgi.Controller): key_data=key_pair['public_key'], metadata=metadata, onset_files=env.get('onset_files', [])) + inst['instance_type'] = flavor_id + inst['image_id'] = requested_image_id - builder = servers_views.get_view_builder(req) - server = builder.build(instances[0], is_detail=False) + builder = self._get_view_builder(req) + server = builder.build(inst, is_detail=True) password = "%s%s" % (server['server']['name'][:4], utils.generate_password(12)) server['server']['adminPass'] = password @@ -437,3 +442,42 @@ class Controller(wsgi.Controller): _("Ramdisk not found for image %(image_id)s") % locals()) return kernel_id, ramdisk_id + + +class ControllerV10(Controller): + def _image_id_from_req_data(self, data): + return data['server']['imageId'] + + def _flavor_id_from_req_data(self, data): + return data['server']['flavorId'] + + def _get_view_builder(self, req): + addresses_builder = nova.api.openstack.views.addresses.ViewBuilderV10() + return nova.api.openstack.views.servers.ViewBuilderV10( + addresses_builder) + + def _get_addresses_view_builder(self, req): + return nova.api.openstack.views.addresses.ViewBuilderV10(req) + + +class ControllerV11(Controller): + def _image_id_from_req_data(self, data): + href = data['server']['imageRef'] + return href.split('/')[-1] + + def _flavor_id_from_req_data(self, data): + href = data['server']['flavorRef'] + return href.split('/')[-1] + + def _get_view_builder(self, req): + base_url = req.application_url + flavor_builder = nova.api.openstack.views.flavors.ViewBuilderV11( + base_url) + image_builder = nova.api.openstack.views.images.ViewBuilderV11( + base_url) + addresses_builder = nova.api.openstack.views.addresses.ViewBuilderV11() + return nova.api.openstack.views.servers.ViewBuilderV11( + addresses_builder, flavor_builder, image_builder) + + def _get_addresses_view_builder(self, req): + return nova.api.openstack.views.addresses.ViewBuilderV11(req) diff --git a/nova/api/openstack/views/addresses.py b/nova/api/openstack/views/addresses.py index 9d392aacea..90c77855b0 100644 --- a/nova/api/openstack/views/addresses.py +++ b/nova/api/openstack/views/addresses.py @@ -19,18 +19,6 @@ from nova import utils from nova.api.openstack import common -def get_view_builder(req): - ''' - A factory method that returns the correct builder based on the version of - the api requested. - ''' - version = common.get_api_version(req) - if version == '1.1': - return ViewBuilder_1_1() - else: - return ViewBuilder_1_0() - - class ViewBuilder(object): ''' Models a server addresses response as a python dictionary.''' @@ -38,14 +26,14 @@ class ViewBuilder(object): raise NotImplementedError() -class ViewBuilder_1_0(ViewBuilder): +class ViewBuilderV10(ViewBuilder): def build(self, inst): private_ips = utils.get_from_path(inst, 'fixed_ip/address') public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') return dict(public=public_ips, private=private_ips) -class ViewBuilder_1_1(ViewBuilder): +class ViewBuilderV11(ViewBuilder): def build(self, inst): private_ips = utils.get_from_path(inst, 'fixed_ip/address') private_ips = [dict(version=4, addr=a) for a in private_ips] diff --git a/nova/api/openstack/views/flavors.py b/nova/api/openstack/views/flavors.py index aa3c2aeb2a..18bd779c09 100644 --- a/nova/api/openstack/views/flavors.py +++ b/nova/api/openstack/views/flavors.py @@ -17,18 +17,6 @@ from nova.api.openstack import common -def get_view_builder(req): - ''' - A factory method that returns the correct builder based on the version of - the api requested. - ''' - version = common.get_api_version(req) - base_url = req.application_url - if version == '1.1': - return ViewBuilder_1_1(base_url) - else: - return ViewBuilder_1_0() - class ViewBuilder(object): def __init__(self): @@ -38,13 +26,9 @@ class ViewBuilder(object): raise NotImplementedError() -class ViewBuilder_1_1(ViewBuilder): +class ViewBuilderV11(ViewBuilder): def __init__(self, base_url): self.base_url = base_url def generate_href(self, flavor_id): return "%s/flavors/%s" % (self.base_url, flavor_id) - - -class ViewBuilder_1_0(ViewBuilder): - pass diff --git a/nova/api/openstack/views/images.py b/nova/api/openstack/views/images.py index 930b464b01..a6c6ad7d10 100644 --- a/nova/api/openstack/views/images.py +++ b/nova/api/openstack/views/images.py @@ -17,18 +17,6 @@ from nova.api.openstack import common -def get_view_builder(req): - ''' - A factory method that returns the correct builder based on the version of - the api requested. - ''' - version = common.get_api_version(req) - base_url = req.application_url - if version == '1.1': - return ViewBuilder_1_1(base_url) - else: - return ViewBuilder_1_0() - class ViewBuilder(object): def __init__(self): @@ -38,13 +26,9 @@ class ViewBuilder(object): raise NotImplementedError() -class ViewBuilder_1_1(ViewBuilder): +class ViewBuilderV11(ViewBuilder): def __init__(self, base_url): self.base_url = base_url def generate_href(self, image_id): return "%s/images/%s" % (self.base_url, image_id) - - -class ViewBuilder_1_0(ViewBuilder): - pass diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 261acfed07..8d47ac757e 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -24,22 +24,6 @@ from nova.api.openstack.views import images as images_view from nova import utils -def get_view_builder(req): - ''' - A factory method that returns the correct builder based on the version of - the api requested. - ''' - version = common.get_api_version(req) - addresses_builder = addresses_view.get_view_builder(req) - if version == '1.1': - flavor_builder = flavors_view.get_view_builder(req) - image_builder = images_view.get_view_builder(req) - return ViewBuilder_1_1(addresses_builder, flavor_builder, - image_builder) - else: - return ViewBuilder_1_0(addresses_builder) - - class ViewBuilder(object): ''' Models a server response as a python dictionary. @@ -76,25 +60,20 @@ class ViewBuilder(object): power_state.FAILED: 'error'} inst_dict = {} - #mapped_keys = dict(status='state', imageId='image_id', - # flavorId='instance_type', name='display_name', id='id') - - mapped_keys = dict(status='state', name='display_name', id='id') - - for k, v in mapped_keys.iteritems(): - inst_dict[k] = inst[v] - - inst_dict['status'] = power_mapping[inst_dict['status']] + inst_dict['id'] = int(inst['id']) + inst_dict['name'] = inst['display_name'] + inst_dict['status'] = power_mapping[inst.get('state')] inst_dict['addresses'] = self.addresses_builder.build(inst) # Return the metadata as a dictionary metadata = {} - for item in inst['metadata']: - metadata[item['key']] = item['value'] + if 'metadata' in inst: + for item in inst['metadata']: + metadata[item['key']] = item['value'] inst_dict['metadata'] = metadata inst_dict['hostId'] = '' - if inst['host']: + if inst.get('host'): inst_dict['hostId'] = hashlib.sha224(inst['host']).hexdigest() self._build_image(inst_dict, inst) @@ -109,24 +88,30 @@ class ViewBuilder(object): raise NotImplementedError() -class ViewBuilder_1_0(ViewBuilder): +class ViewBuilderV10(ViewBuilder): def _build_image(self, response, inst): - response["imageId"] = inst["image_id"] + if inst.get('image_id') != None: + response['imageId'] = inst['image_id'] def _build_flavor(self, response, inst): - response["flavorId"] = inst["instance_type"] + if inst.get('instance_type') != None: + response['flavorId'] = inst['instance_type'] -class ViewBuilder_1_1(ViewBuilder): +class ViewBuilderV11(ViewBuilder): def __init__(self, addresses_builder, flavor_builder, image_builder): ViewBuilder.__init__(self, addresses_builder) self.flavor_builder = flavor_builder self.image_builder = image_builder def _build_image(self, response, inst): + if inst.get('image_id') == None: + return image_id = inst["image_id"] response["imageRef"] = self.image_builder.generate_href(image_id) def _build_flavor(self, response, inst): + if inst.get('instance_type') == None: + return flavor_id = inst["instance_type"] response["flavorRef"] = self.flavor_builder.generate_href(flavor_id) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index a3968b57bf..370eb68cb3 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -76,14 +76,18 @@ def fake_wsgi(self, req): return self.application -def wsgi_app(inner_application=None): - if not inner_application: - inner_application = openstack.APIRouter() +def wsgi_app(inner_app10=None, inner_app11=None): + if not inner_app10: + inner_app10 = openstack.APIRouterV10() + if not inner_app11: + inner_app11 = openstack.APIRouterV11() mapper = urlmap.URLMap() - api = openstack.FaultWrapper(auth.AuthMiddleware( - ratelimiting.RateLimitingMiddleware(inner_application))) - mapper['/v1.0'] = api - mapper['/v1.1'] = api + api10 = openstack.FaultWrapper(auth.AuthMiddleware( + ratelimiting.RateLimitingMiddleware(inner_app10))) + api11 = openstack.FaultWrapper(auth.AuthMiddleware( + ratelimiting.RateLimitingMiddleware(inner_app11))) + mapper['/v1.0'] = api10 + mapper['/v1.1'] = api11 mapper['/'] = openstack.FaultWrapper(openstack.Versions()) return mapper diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index aaaa4e4159..b7f0dfbe5c 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -82,8 +82,7 @@ class Test(test.TestCase): self.assertEqual(result.headers['X-Storage-Url'], "") token = result.headers['X-Auth-Token'] - self.stubs.Set(nova.api.openstack, 'APIRouter', - fakes.FakeRouter) + self.stubs.Set(nova.api.openstack, 'APIRouterV10', fakes.FakeRouter) req = webob.Request.blank('/v1.0/fake') req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) @@ -189,8 +188,7 @@ class TestLimiter(test.TestCase): self.assertEqual(len(result.headers['X-Auth-Token']), 40) token = result.headers['X-Auth-Token'] - self.stubs.Set(nova.api.openstack, 'APIRouter', - fakes.FakeRouter) + self.stubs.Set(nova.api.openstack, 'APIRouterV10', fakes.FakeRouter) req = webob.Request.blank('/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 27d174fe96..0116bbed19 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -158,7 +158,7 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) - self.assertEqual(res_dict['server']['id'], '1') + self.assertEqual(res_dict['server']['id'], 1) self.assertEqual(res_dict['server']['name'], 'server1') def test_get_server_by_id_with_addresses(self): @@ -169,7 +169,7 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) - self.assertEqual(res_dict['server']['id'], '1') + self.assertEqual(res_dict['server']['id'], 1) self.assertEqual(res_dict['server']['name'], 'server1') addresses = res_dict['server']['addresses'] self.assertEqual(len(addresses["public"]), len(public)) @@ -177,7 +177,7 @@ class ServersTest(test.TestCase): self.assertEqual(len(addresses["private"]), 1) self.assertEqual(addresses["private"][0], private) - def test_get_server_by_id_with_addresses_v1_1(self): + def test_get_server_by_id_with_addresses_v11(self): private = "192.168.0.3" public = ["1.2.3.4"] new_return_server = return_server_with_addresses(private, public) @@ -186,7 +186,7 @@ class ServersTest(test.TestCase): req.environ['api.version'] = '1.1' res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) - self.assertEqual(res_dict['server']['id'], '1') + self.assertEqual(res_dict['server']['id'], 1) self.assertEqual(res_dict['server']['name'], 'server1') addresses = res_dict['server']['addresses'] self.assertEqual(len(addresses["public"]), len(public)) @@ -273,13 +273,13 @@ class ServersTest(test.TestCase): "get_image_id_from_image_hash", image_id_from_hash) body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, + name='server_test', imageId=3, flavorId=2, metadata={'hello': 'world', 'open': 'stack'}, personality={})) req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) - req.headers["Content-Type"] = "application/json" + req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) @@ -287,8 +287,67 @@ class ServersTest(test.TestCase): self.assertEqual('serv', server['adminPass'][:4]) self.assertEqual(16, len(server['adminPass'])) self.assertEqual('server_test', server['name']) - self.assertEqual('1', server['id']) + self.assertEqual(1, server['id']) + self.assertEqual(2, server['flavorId']) + self.assertEqual(3, server['imageId']) + self.assertEqual(res.status_int, 200) + def test_create_instance_v11(self): + def instance_create(context, inst): + return {'id': '1', 'display_name': 'server_test'} + + def server_update(context, id, params): + return instance_create(context, id) + + def fake_method(*args, **kwargs): + pass + + def project_get_network(context, user_id): + return dict(id='1', host='localhost') + + def queue_get_for(context, *args): + return 'network_topic' + + def kernel_ramdisk_mapping(*args, **kwargs): + return (1, 1) + + def image_id_from_hash(*args, **kwargs): + return 2 + + self.stubs.Set(nova.db.api, 'project_get_network', project_get_network) + self.stubs.Set(nova.db.api, 'instance_create', instance_create) + self.stubs.Set(nova.rpc, 'cast', fake_method) + self.stubs.Set(nova.rpc, 'call', fake_method) + self.stubs.Set(nova.db.api, 'instance_update', + server_update) + self.stubs.Set(nova.db.api, 'queue_get_for', queue_get_for) + self.stubs.Set(nova.network.manager.VlanManager, 'allocate_fixed_ip', + fake_method) + self.stubs.Set(nova.api.openstack.servers.Controller, + "_get_kernel_ramdisk_from_image", kernel_ramdisk_mapping) + self.stubs.Set(nova.api.openstack.common, + "get_image_id_from_image_hash", image_id_from_hash) + + imageRef = 'http://localhost/v1.1/images/2' + flavorRef = 'http://localhost/v1.1/flavors/3' + body = dict(server=dict( + name='server_test', imageRef=imageRef, flavorRef=flavorRef, + metadata={'hello': 'world', 'open': 'stack'}, + personality={})) + req = webob.Request.blank('/v1.1/servers') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + + server = json.loads(res.body)['server'] + self.assertEqual('serv', server['adminPass'][:4]) + self.assertEqual(16, len(server['adminPass'])) + self.assertEqual('server_test', server['name']) + self.assertEqual(1, server['id']) + self.assertEqual(flavorRef, server['flavorRef']) + self.assertEqual(imageRef, server['imageRef']) self.assertEqual(res.status_int, 200) def test_update_no_body(self): From 05ca6e24d4a3cf64bbe371f1c9c74088110eba68 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Thu, 17 Mar 2011 04:32:24 -0400 Subject: [PATCH 034/101] Setting the api verion in the request in the auth middle is no longer needed. Also, common.get_api_version is no longer needed. As Eric Day noted, having versioned controllers will make that unnecessary. --- nova/api/openstack/auth.py | 2 -- nova/api/openstack/common.py | 4 ---- 2 files changed, 6 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6f1cf5e63f..4c6b58efff 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -69,8 +69,6 @@ class AuthMiddleware(wsgi.Middleware): return faults.Fault(webob.exc.HTTPUnauthorized()) req.environ['nova.context'] = context.RequestContext(user, account) - version = req.path.split('/')[1].replace('v', '') - req.environ['api.version'] = version return self.application def has_authentication(self, req): diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index d6679de010..74ac210245 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -74,7 +74,3 @@ def get_image_id_from_image_hash(image_service, context, image_hash): if abs(hash(image_id)) == int(image_hash): return image_id raise exception.NotFound(image_hash) - - -def get_api_version(req): - return req.environ.get('api.version') From aa13754d04c17ae9985017e22ae4f68916bc2781 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 10:03:47 -0500 Subject: [PATCH 035/101] Foo --- nova/compute/manager.py | 1 - nova/virt/xenapi/vmops.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3135d58019..b8c3c24cdf 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -547,7 +547,6 @@ class ComputeManager(manager.Manager): vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) - instance_ref = self.db.instance_get(context, instance_id) self.driver.finish_resize(instance_ref, disk_info) self.db.migration_update(context, migration_id, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 9719e05b99..b5003f0f86 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -307,7 +307,7 @@ class VMOps(object): template_vdi_uuids = template_vm_ref = None try: # transfer the base copy - template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) + template_vm_ref, template_vdi_uuids = selimage._get_snapshot(instance) base_copy_uuid = template_vdi_uuids['image'] vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) @@ -368,6 +368,7 @@ class VMOps(object): """Resize a running instance by changing it's RAM and disk size """ #TODO(mdietz): this will need to be adjusted for swap later #The new disk size must be in bytes + new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB") % (vdi_uuid, instance.name, instance.local_gb)) From 4f1f5bb1ed2cbb57e9ba8ea481ae31c0e6acc7bd Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 11:03:07 -0500 Subject: [PATCH 036/101] refactoring --- nova/compute/manager.py | 5 +---- nova/virt/xenapi/vmops.py | 22 +++++++++++++++++----- nova/virt/xenapi_conn.py | 10 +++++----- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b8c3c24cdf..6b784f1e34 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -463,7 +463,7 @@ class ComputeManager(manager.Manager): vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) - self.driver._start(instance_ref) + self.driver.revert_resize(instance_ref) self.db.migration_update(context, migration_id, {'status': 'reverted'}) @@ -514,8 +514,6 @@ class ComputeManager(manager.Manager): self.db.migration_update(context, migration_id, {'status': 'post-migrating', }) - - service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, @@ -536,7 +534,6 @@ class ComputeManager(manager.Manager): migration_ref = self.db.migration_get(context, migration_id) instance_ref = self.db.instance_get(context, migration_ref['instance_id']) - #TODO(mdietz): apply the rest of the instance_type attributes going #after they're supported instance_type = self.db.instance_type_get_by_flavor_id(context, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b5003f0f86..ee99a99181 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -62,6 +62,17 @@ class VMOps(object): vm_refs.append(vm_rec["name_label"]) return vm_refs + def revert_resize(self, instance): + vm_ref = VMHelper.lookup(self._session, instance.name) + self._start(instance, vm_ref) + + def finish_resize(self, instance, disk_info): + vdi_uuid = self._vmops.link_disks(instance, disk_info['base_copy'], + disk_info['cow']) + vm_ref = self._create_vm(instance, vdi_uuid) + self.resize_instance(instance, vdi_uuid) + self._spawn(instance, vm_ref) + def _start(self, instance, vm_ref=None): """Power on a VM instance""" if not vm_ref: @@ -307,7 +318,8 @@ class VMOps(object): template_vdi_uuids = template_vm_ref = None try: # transfer the base copy - template_vm_ref, template_vdi_uuids = selimage._get_snapshot(instance) + template_vm_ref, template_vdi_uuids = \ + self.image._get_snapshot(instance) base_copy_uuid = template_vdi_uuids['image'] vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) @@ -370,8 +382,8 @@ class VMOps(object): #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB") % (vdi_uuid, - instance.name, instance.local_gb)) + LOG.debug(_("Resizpng VDI %s for instance %s. Expanding to %sGB") % + (vdi_uuid, instance.name, instance.local_gb)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) LOG.debug(_("Resize instance %s complete") % (instance.name)) @@ -451,8 +463,8 @@ class VMOps(object): state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: instance_name = instance.name - LOG.warn(_("VM %(instance_name)s already halted, skipping shutdown...") % - locals()) + LOG.warn(_("VM %(instance_name)s already halted," + "skipping shutdown...") % locals()) return instance_id = instance.id diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 2b0f82a4a1..da2fb51f11 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -164,13 +164,13 @@ class XenAPIConnection(object): """Create VM instance""" self._vmops.spawn(instance) + def revert_resize(self, instance): + """Reverts a resize, powering back on the instance""" + self._vmops.revert_resize(instance) + def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" - vdi_uuid = self._vmops.link_disks(instance, disk_info['base_copy'], - disk_info['cow']) - vm_ref = self._vmops._create_vm(instance, vdi_uuid) - self._vmops.resize_instance(instance, vdi_uuid) - self._vmops._spawn(instance, vm_ref) + self._vmops.finish_resize(instance, disk_info) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ From 1ffef31839f3c1f4386d5df834af6d53483c09ed Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 11:16:59 -0500 Subject: [PATCH 037/101] oh come on --- nova/virt/xenapi/vmops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ee99a99181..b6bcc60ea7 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -318,8 +318,7 @@ class VMOps(object): template_vdi_uuids = template_vm_ref = None try: # transfer the base copy - template_vm_ref, template_vdi_uuids = \ - self.image._get_snapshot(instance) + template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) base_copy_uuid = template_vdi_uuids['image'] vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) From e79eaca86c4073cc8bc6c59be83d0f1bf5e2cea4 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 12:20:22 -0400 Subject: [PATCH 038/101] glance image service show testcases --- nova/image/glance.py | 14 ++++++++ nova/tests/image/test_glance.py | 64 +++++++++++++++++++++++++++++---- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 3b448db4b5..d0c191ea1e 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -18,6 +18,8 @@ from __future__ import absolute_import +import datetime as dt + from glance.common import exception as glance_exception from nova import exception @@ -60,6 +62,18 @@ class GlanceImageService(service.BaseImageService): """ try: image = self.client.get_image_meta(image_id) + if 'created_at' in image: + image['created_at'] = \ + dt.datetime.strptime(image['created_at'], + "%Y-%m-%dT%H:%M:%S.%f") + if 'updated_at' in image: + image['updated_at'] = \ + dt.datetime.strptime(image['updated_at'], + "%Y-%m-%dT%H:%M:%S.%f") + if 'deleted_at' in image and image['deleted_at'] is not None: + image['deleted_at'] = \ + dt.datetime.strptime(image['deleted_at'], + "%Y-%m-%dT%H:%M:%S.%f") except glance_exception.NotFound: raise exception.NotFound return image diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index b568f593de..971a32a178 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -1,3 +1,4 @@ +import datetime as dt import unittest from nova.image import glance @@ -5,14 +6,63 @@ from nova.image import glance class StubGlanceClient(object): def __init__(self, images): - self._images = images + self.images = images - def get_image_meta(id): - return self._images[id] + def get_image_meta(self, id): + return self.images[id] + + def get_images_detailed(self): + return self.images class TestGlance(unittest.TestCase): - def test(self): - images = {'xyz': "image"} - client = StubGlanceClient(images) - service = glance.GlanceImageService(client) + def setUp(self): + self.client = StubGlanceClient(None) + self.service = glance.GlanceImageService(self.client) + + def test_show_passes_through_to_client(self): + self.client.images = {'xyz': "image"} + self.assertEqual(self.service.show({}, 'xyz'), "image") + + def test_detail_passes_through_to_client(self): + self.client.images = "these are the images" + self.assertEqual(self.service.detail({}), self.client.images) + + def test_show_makes_create_datetimes(self): + create_time = dt.datetime.utcnow() + self.client.images = {'xyz': { + 'id': "id", + 'name': "my awesome image", + 'created_at': create_time.isoformat(), + }} + actual = self.service.show({}, 'xyz') + self.assertEqual(actual['created_at'], create_time) + + def test_show_makes_update_datetimes(self): + update_time = dt.datetime.utcnow() + self.client.images = {'abc': { + 'id': "id", + 'name': "my okay image", + 'updated_at': update_time.isoformat(), + }} + actual = self.service.show({}, 'abc') + self.assertEqual(actual['updated_at'], update_time) + + def test_show_makes_delete_datetimes(self): + delete_time = dt.datetime.utcnow() + self.client.images = {'123': { + 'id': "123", + 'name': "my lame image", + 'deleted_at': delete_time.isoformat(), + }} + actual = self.service.show({}, '123') + self.assertEqual(actual['deleted_at'], delete_time) + + def test_show_handles_deleted_at_none(self): + self.client.images = {'747': { + 'id': "747", + 'name': "not deleted", + 'deleted_at': None, + }} + actual = self.service.show({}, '747') + self.assertEqual(actual['deleted_at'], None) From d6ae8e4c2f6011497b1db23fcbafb23b663f924d Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 11:24:24 -0500 Subject: [PATCH 039/101] Foo --- nova/compute/manager.py | 1 + nova/virt/xenapi/vmops.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6b784f1e34..186b6f6a5d 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -544,6 +544,7 @@ class ComputeManager(manager.Manager): vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) + instance_ref = self.db.instance_get(context, instance_id) self.driver.finish_resize(instance_ref, disk_info) self.db.migration_update(context, migration_id, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b6bcc60ea7..326d43aa91 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -67,7 +67,7 @@ class VMOps(object): self._start(instance, vm_ref) def finish_resize(self, instance, disk_info): - vdi_uuid = self._vmops.link_disks(instance, disk_info['base_copy'], + vdi_uuid = self.link_disks(instance, disk_info['base_copy'], disk_info['cow']) vm_ref = self._create_vm(instance, vdi_uuid) self.resize_instance(instance, vdi_uuid) From b135bc23cca1494049dd9978cb18b52f2b4d99c7 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 12:30:32 -0400 Subject: [PATCH 040/101] refactor to simpler implementation --- nova/image/glance.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index d0c191ea1e..188b6e588f 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -62,22 +62,25 @@ class GlanceImageService(service.BaseImageService): """ try: image = self.client.get_image_meta(image_id) - if 'created_at' in image: - image['created_at'] = \ - dt.datetime.strptime(image['created_at'], - "%Y-%m-%dT%H:%M:%S.%f") - if 'updated_at' in image: - image['updated_at'] = \ - dt.datetime.strptime(image['updated_at'], - "%Y-%m-%dT%H:%M:%S.%f") - if 'deleted_at' in image and image['deleted_at'] is not None: - image['deleted_at'] = \ - dt.datetime.strptime(image['deleted_at'], - "%Y-%m-%dT%H:%M:%S.%f") except glance_exception.NotFound: raise exception.NotFound + return self._convert_timestamps_to_datetimes(image) + + def _convert_timestamps_to_datetimes(self, image): + """ + Returns image with known timestamp fields converted to datetime objects + """ + for attr in ['created_at', 'updated_at', 'deleted_at']: + if attr in image and image[attr] is not None: + image[attr] = self._parse_glance_iso8601_timestamp(image[attr]) return image + def _parse_glance_iso8601_timestamp(self, timestamp): + """ + Parse a subset of iso8601 timestamps into datetime objects + """ + return dt.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") + def show_by_name(self, context, name): """ Returns a dict containing image data for the given name. From 686e113188aaf8195aed7bea8bf70c21b6bff498 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 12:04:49 -0500 Subject: [PATCH 041/101] Mapping the resize status --- nova/api/openstack/servers.py | 8 +++++++- nova/compute/manager.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 47ed254ec4..59234b0de7 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -61,7 +61,13 @@ def _translate_detail_keys(inst): for k, v in mapped_keys.iteritems(): inst_dict[k] = inst[v] - inst_dict['status'] = power_mapping[inst_dict['status']] + context = req.environ['nova.context'].elevated() + migration = self.db.migrate_get_by_instance_and_status(context, + inst['id'], 'finished') + if migration: + inst_dict['status'] = 'resize-confirm' + else + inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) # grab single private fixed ip diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 186b6f6a5d..7993298b95 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -544,6 +544,8 @@ class ComputeManager(manager.Manager): vcpus=instance_type['vcpus'], local_gb=instance_type['local_gb'])) + # reload the updated instance ref + # FIXME: is there reload functionality? instance_ref = self.db.instance_get(context, instance_id) self.driver.finish_resize(instance_ref, disk_info) From 3afeb8466fa9f005edc9da182b1e0af6ffb00ade Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 12:05:43 -0500 Subject: [PATCH 042/101] Mapping the resize status --- nova/api/openstack/servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 59234b0de7..fd835c2474 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -66,7 +66,7 @@ def _translate_detail_keys(inst): inst['id'], 'finished') if migration: inst_dict['status'] = 'resize-confirm' - else + else: inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) From 3ee835c60d2b43086b1e324501025d1f0221da27 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 13:50:41 -0400 Subject: [PATCH 043/101] handle timestamps in glance service detail --- nova/image/glance.py | 3 ++- nova/tests/image/test_glance.py | 27 ++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 188b6e588f..7706a42e49 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -54,7 +54,8 @@ class GlanceImageService(service.BaseImageService): """ Calls out to Glance for a list of detailed image information """ - return self.client.get_images_detailed() + for image in self.client.get_images_detailed(): + yield self._convert_timestamps_to_datetimes(image) def show(self, context, image_id): """ diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 971a32a178..16fe0e7c05 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -12,7 +12,7 @@ class StubGlanceClient(object): return self.images[id] def get_images_detailed(self): - return self.images + return self.images.itervalues() class TestGlance(unittest.TestCase): @@ -25,8 +25,8 @@ class TestGlance(unittest.TestCase): self.assertEqual(self.service.show({}, 'xyz'), "image") def test_detail_passes_through_to_client(self): - self.client.images = "these are the images" - self.assertEqual(self.service.detail({}), self.client.images) + self.client.images = {1: "an image"} + self.assertEqual(list(self.service.detail({})), ["an image"]) def test_show_makes_create_datetimes(self): create_time = dt.datetime.utcnow() @@ -66,3 +66,24 @@ class TestGlance(unittest.TestCase): }} actual = self.service.show({}, '747') self.assertEqual(actual['deleted_at'], None) + + def test_detail_handles_timestamps(self): + now = dt.datetime.utcnow() + image1 = { + 'id': 1, + 'name': 'image 1', + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), + 'deleted_at': None, + } + image2 = { + 'id': 2, + 'name': 'image 2', + 'deleted_at': now.isoformat(), + } + self.client.images = {1: image1, 2: image2} + i1, i2 = self.service.detail({}) + self.assertEqual(i1['created_at'], now) + self.assertEqual(i1['updated_at'], now) + self.assertEqual(i1['deleted_at'], None) + self.assertEqual(i2['deleted_at'], now) From 66c237a4d321887830e5282781870525abf00365 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 14:04:31 -0400 Subject: [PATCH 044/101] teach glance image server get to handle timestamps --- nova/image/glance.py | 2 +- nova/tests/image/test_glance.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 7706a42e49..f725fe176b 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -108,7 +108,7 @@ class GlanceImageService(service.BaseImageService): raise exception.NotFound for chunk in image_chunks: data.write(chunk) - return metadata + return self._convert_timestamps_to_datetimes(metadata) def create(self, context, metadata, data=None): """ diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 16fe0e7c05..1e6c452196 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -14,7 +14,17 @@ class StubGlanceClient(object): def get_images_detailed(self): return self.images.itervalues() -class TestGlance(unittest.TestCase): + def get_image(self, id): + return self.images[id], [] + + +class NullWriter(object): + + def write(self, *arg, **kwargs): + pass + + +class TestGlanceImageServiceDatetimes(unittest.TestCase): def setUp(self): self.client = StubGlanceClient(None) @@ -87,3 +97,21 @@ class TestGlance(unittest.TestCase): self.assertEqual(i1['updated_at'], now) self.assertEqual(i1['deleted_at'], None) self.assertEqual(i2['deleted_at'], now) + + def test_get_handles_timestamps(self): + now = dt.datetime.utcnow() + self.client.images = {'abcd': { + 'id': 'abcd', + 'name': 'nifty image', + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), + 'deleted_at': now.isoformat(), + }} + actual = self.service.get({}, 'abcd', NullWriter()) + for attr in ('created_at', 'updated_at', 'deleted_at'): + self.assertEqual(actual[attr], now) + + def test_get_handles_deleted_at_none(self): + self.client.images = {'abcd': {'deleted_at': None}} + actual = self.service.get({}, 'abcd', NullWriter()) + self.assertEqual(actual['deleted_at'], None) From c8e474d04dce462650c2a9f57cbcb106ce3ef0c9 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 14:05:08 -0400 Subject: [PATCH 045/101] pep8 --- nova/tests/image/test_glance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 1e6c452196..9b17cf2613 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -3,6 +3,7 @@ import unittest from nova.image import glance + class StubGlanceClient(object): def __init__(self, images): From 4334ca9d6b0ac8a9b2edb1fbcbf0bc4df28b2961 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 15:04:28 -0400 Subject: [PATCH 046/101] get api openstack test_images working --- nova/image/glance.py | 7 ++++--- nova/tests/api/openstack/test_images.py | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index f725fe176b..55dc5488d6 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -42,7 +42,8 @@ class GlanceImageService(service.BaseImageService): def __init__(self, client=None): if client is None: self.client = GlanceClient(FLAGS.glance_host, FLAGS.glance_port) - self.client = client + else: + self.client = client def index(self, context): """ @@ -54,8 +55,8 @@ class GlanceImageService(service.BaseImageService): """ Calls out to Glance for a list of detailed image information """ - for image in self.client.get_images_detailed(): - yield self._convert_timestamps_to_datetimes(image) + return [self._convert_timestamps_to_datetimes(image) + for image in self.client.get_images_detailed()] def show(self, context, image_id): """ diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 76f7589294..47dd11e5b6 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -182,8 +182,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): {'id': '23g2ogk23k4hhkk4k42l', 'imageId': '23g2ogk23k4hhkk4k42l', 'name': 'public image #1', - 'created_at': str(datetime.datetime.utcnow()), - 'updated_at': str(datetime.datetime.utcnow()), + 'created_at': datetime.datetime.utcnow().isoformat(), + 'updated_at': datetime.datetime.utcnow().isoformat(), 'deleted_at': None, 'deleted': False, 'is_public': True, @@ -192,8 +192,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): {'id': 'slkduhfas73kkaskgdas', 'imageId': 'slkduhfas73kkaskgdas', 'name': 'public image #2', - 'created_at': str(datetime.datetime.utcnow()), - 'updated_at': str(datetime.datetime.utcnow()), + 'created_at': datetime.datetime.utcnow().isoformat(), + 'updated_at': datetime.datetime.utcnow().isoformat(), 'deleted_at': None, 'deleted': False, 'is_public': True, From 25c407b6ade499dd0bdd470e7fd46682c34a98b7 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Thu, 17 Mar 2011 19:13:09 +0000 Subject: [PATCH 047/101] Get the migration out --- nova/api/openstack/servers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index fd835c2474..601a685087 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -20,6 +20,8 @@ import traceback from webob import exc from nova import compute +from nova import context +from nova import db from nova import exception from nova import flags from nova import log as logging @@ -61,12 +63,12 @@ def _translate_detail_keys(inst): for k, v in mapped_keys.iteritems(): inst_dict[k] = inst[v] - context = req.environ['nova.context'].elevated() - migration = self.db.migrate_get_by_instance_and_status(context, - inst['id'], 'finished') - if migration: + ctxt = context.get_admin_context() + try: + migration = db.migration_get_by_instance_and_status(ctxt, + inst['id'], 'finished') inst_dict['status'] = 'resize-confirm' - else: + except Exception, e: inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) From 2f1a1d293915cde6e15c85e0bb43fb21ae26f7b0 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 17 Mar 2011 15:29:54 -0400 Subject: [PATCH 048/101] handle create and update requests, and update the base image service documentation to reflect the (defacto) behavior --- nova/image/glance.py | 7 +++-- nova/image/service.py | 4 +-- nova/tests/image/test_glance.py | 54 ++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 55dc5488d6..fbb578585a 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -118,7 +118,8 @@ class GlanceImageService(service.BaseImageService): :raises AlreadyExists if the image already exist. """ - return self.client.add_image(metadata, data) + return self._convert_timestamps_to_datetimes( + self.client.add_image(metadata, data)) def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. @@ -127,10 +128,10 @@ class GlanceImageService(service.BaseImageService): """ try: - result = self.client.update_image(image_id, metadata, data) + metadata = self.client.update_image(image_id, metadata, data) except glance_exception.NotFound: raise exception.NotFound - return result + return self._convert_timestamps_to_datetimes(metadata) def delete(self, context, image_id): """ diff --git a/nova/image/service.py b/nova/image/service.py index 78d8f33e97..e907381c98 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -88,7 +88,7 @@ class BaseImageService(object): def create(self, context, metadata, data=None): """ - Store the image metadata and data and return the new image id. + Store the image metadata and data and return the new image metadata. :raises AlreadyExists if the image already exist. @@ -96,7 +96,7 @@ class BaseImageService(object): raise NotImplementedError def update(self, context, image_id, metadata, data=None): - """Update the given image with the new metadata and data. + """Update the given image metadata and data and return the metadata :raises NotFound if the image does not exist. diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 9b17cf2613..6e94aa9097 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -6,8 +6,10 @@ from nova.image import glance class StubGlanceClient(object): - def __init__(self, images): + def __init__(self, images, add_response=None, update_response=None): self.images = images + self.add_response = add_response + self.update_response = update_response def get_image_meta(self, id): return self.images[id] @@ -18,6 +20,12 @@ class StubGlanceClient(object): def get_image(self, id): return self.images[id], [] + def add_image(self, metadata, data): + return self.add_response + + def update_image(self, image_id, metadata, data): + return self.update_response + class NullWriter(object): @@ -116,3 +124,47 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.client.images = {'abcd': {'deleted_at': None}} actual = self.service.get({}, 'abcd', NullWriter()) self.assertEqual(actual['deleted_at'], None) + + def test_create_handles_timestamps(self): + now = dt.datetime.utcnow() + self.client.add_response = { + 'id': 'abcd', + 'name': 'blah', + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), + 'deleted_at': now.isoformat(), + } + actual = self.service.create({}, {}) + for attr in ('created_at', 'updated_at', 'deleted_at'): + self.assertEqual(actual[attr], now) + + def test_create_handles_deleted_at_none(self): + self.client.add_response = { + 'id': 'abcd', + 'name': 'blah', + 'deleted_at': None, + } + actual = self.service.create({}, {}) + self.assertEqual(actual['deleted_at'], None) + + def test_update_handles_timestamps(self): + now = dt.datetime.utcnow() + self.client.update_response = { + 'id': 'abcd', + 'name': 'blah', + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), + 'deleted_at': now.isoformat(), + } + actual = self.service.update({}, 'dummy_id', {}) + for attr in ('created_at', 'updated_at', 'deleted_at'): + self.assertEqual(actual[attr], now) + + def test_create_handles_deleted_at_none(self): + self.client.update_response = { + 'id': 'abcd', + 'name': 'blah', + 'deleted_at': None, + } + actual = self.service.update({}, 'dummy_id', {}) + self.assertEqual(actual['deleted_at'], None) From e138e0836922ee0608feefbff5e4e5d03ad14197 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Thu, 17 Mar 2011 16:02:37 -0400 Subject: [PATCH 049/101] Now returns a 400 for a create server request with invalid hrefs for imageRef/flavorRef values. Also added tests. --- nova/api/openstack/common.py | 12 ++++- nova/api/openstack/servers.py | 5 +- nova/tests/api/openstack/test_servers.py | 66 ++++++++++-------------- 3 files changed, 38 insertions(+), 45 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 74ac210245..b224cbfb45 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -15,9 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. -import webob.exc - +import re from nova import exception +from webob import exc +import webob.exc def limited(items, request, max_limit=1000): @@ -74,3 +75,10 @@ def get_image_id_from_image_hash(image_service, context, image_hash): if abs(hash(image_id)) == int(image_hash): return image_id raise exception.NotFound(image_hash) + + +def get_id_from_href(href): + m = re.match(r'http.+/.+/(\d)+$', href) + if not m: + raise exc.HTTPBadRequest(_('could not parse id from href')) + return int(m.group(1)) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f03225b552..6f25d10bd9 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -512,7 +512,6 @@ class Controller(wsgi.Controller): return kernel_id, ramdisk_id - class ControllerV10(Controller): def _image_id_from_req_data(self, data): return data['server']['imageId'] @@ -532,11 +531,11 @@ class ControllerV10(Controller): class ControllerV11(Controller): def _image_id_from_req_data(self, data): href = data['server']['imageRef'] - return href.split('/')[-1] + return common.get_id_from_href(href) def _flavor_id_from_req_data(self, data): href = data['server']['flavorRef'] - return href.split('/')[-1] + return common.get_id_from_href(href) def _get_view_builder(self, req): base_url = req.application_url diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 846af5c3a4..6e78db9daf 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -239,7 +239,7 @@ class ServersTest(test.TestCase): servers = json.loads(res.body)['servers'] self.assertEqual([s['id'] for s in servers], [1, 2]) - def _test_create_instance_helper(self): + def _setup_for_create_instance(self): """Shared implementation for tests below that create instance""" def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} @@ -276,6 +276,9 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.api.openstack.common, "get_image_id_from_image_hash", image_id_from_hash) + def _test_create_instance_helper(self): + self._setup_for_create_instance() + body = dict(server=dict( name='server_test', imageId=3, flavorId=2, metadata={'hello': 'world', 'open': 'stack'}, @@ -296,41 +299,15 @@ class ServersTest(test.TestCase): self.assertEqual(3, server['imageId']) self.assertEqual(res.status_int, 200) + def test_create_instance(self): + self._test_create_instance_helper() + + def test_create_instance_no_key_pair(self): + fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) + self._test_create_instance_helper() + def test_create_instance_v11(self): - def instance_create(context, inst): - return {'id': '1', 'display_name': 'server_test'} - - def server_update(context, id, params): - return instance_create(context, id) - - def fake_method(*args, **kwargs): - pass - - def project_get_network(context, user_id): - return dict(id='1', host='localhost') - - def queue_get_for(context, *args): - return 'network_topic' - - def kernel_ramdisk_mapping(*args, **kwargs): - return (1, 1) - - def image_id_from_hash(*args, **kwargs): - return 2 - - self.stubs.Set(nova.db.api, 'project_get_network', project_get_network) - self.stubs.Set(nova.db.api, 'instance_create', instance_create) - self.stubs.Set(nova.rpc, 'cast', fake_method) - self.stubs.Set(nova.rpc, 'call', fake_method) - self.stubs.Set(nova.db.api, 'instance_update', - server_update) - self.stubs.Set(nova.db.api, 'queue_get_for', queue_get_for) - self.stubs.Set(nova.network.manager.VlanManager, 'allocate_fixed_ip', - fake_method) - self.stubs.Set(nova.api.openstack.servers.Controller, - "_get_kernel_ramdisk_from_image", kernel_ramdisk_mapping) - self.stubs.Set(nova.api.openstack.common, - "get_image_id_from_image_hash", image_id_from_hash) + self._setup_for_create_instance() imageRef = 'http://localhost/v1.1/images/2' flavorRef = 'http://localhost/v1.1/flavors/3' @@ -354,12 +331,21 @@ class ServersTest(test.TestCase): self.assertEqual(imageRef, server['imageRef']) self.assertEqual(res.status_int, 200) - def test_create_instance(self): - self._test_create_instance_helper() + def test_create_instance_v11_bad_href(self): + self._setup_for_create_instance() - def test_create_instance_no_key_pair(self): - fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) - self._test_create_instance_helper() + imageRef = 'http://localhost/v1.1/images/asdf' + flavorRef = 'http://localhost/v1.1/flavors/3' + body = dict(server=dict( + name='server_test', imageRef=imageRef, flavorRef=flavorRef, + metadata={'hello': 'world', 'open': 'stack'}, + personality={})) + req = webob.Request.blank('/v1.1/servers') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') From f7d5dea09568c6440918264d97ecdbcc316c0ec4 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Mar 2011 15:31:48 -0500 Subject: [PATCH 050/101] pep8 --- nova/api/openstack/servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index b0e3552326..0504504575 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -69,7 +69,7 @@ def _translate_detail_keys(inst): ctxt = context.get_admin_context() try: migration = db.migration_get_by_instance_and_status(ctxt, - inst['id'], 'finished') + inst['id'], 'finished') inst_dict['status'] = 'resize-confirm' except Exception, e: inst_dict['status'] = power_mapping[inst_dict['status']] From b605b53e4b652e0a3f364d505b5fd7240fd4ea36 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Thu, 17 Mar 2011 20:44:15 +0000 Subject: [PATCH 051/101] Test changes --- nova/tests/api/openstack/test_servers.py | 22 ++++++++++++---------- nova/tests/xenapi/stubs.py | 7 +++++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 03e00af2a3..14b72e097f 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -491,16 +491,6 @@ class ServersTest(test.TestCase): req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - def test_server_resize(self): - body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, metadata={}, - personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') - req.method = 'POST' - req.content_type = 'application/json' - req.body = json.dumps(body) - res = req.get_response(fakes.wsgi_app()) - def test_delete_server_instance(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'DELETE' @@ -556,6 +546,18 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 400) + def test_resized_server_has_correct_status(self): + req = self.webreq('/1', 'GET', dict(resize=dict(flavorId=3))) + def fake_migration_get(*args): + return {} + + self.stubs.Set(nova.db, 'migration_get_by_instance_and_status', + fake_migration_get) + res = req.get_response(fakes.wsgi_app()) + body = json.loads(res.body) + self.assertEqual(body['server']['status'], 'resize-confirm') + + def test_confirm_resize_server(self): req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 70d46a1fbb..7f9706a3dd 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -228,6 +228,9 @@ class FakeSessionForMigrationTests(fake.SessionBase): def VDI_get_by_uuid(*args): return 'hurr' + def VDI_resize_online(*args): + pass + def VM_start(self, _1, ref, _2, _3): vm = fake.get_record('VM', ref) if vm['power_state'] != 'Halted': @@ -240,7 +243,7 @@ class FakeSessionForMigrationTests(fake.SessionBase): def stub_out_migration_methods(stubs): def fake_get_snapshot(self, instance): - return 'foo', 'bar' + return 'vm_ref', dict(image='foo', snap='bar') @classmethod def fake_get_vdi(cls, session, vm_ref): @@ -249,7 +252,7 @@ def stub_out_migration_methods(stubs): vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) return vdi_ref, {'uuid': vdi_rec['uuid'], } - def fake_shutdown(self, inst, vm, method='clean'): + def fake_shutdown(self, inst, vm, hard=True): pass @classmethod From 8d5ffa079e768adec969a4e8ab540c24a7faaaa6 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Thu, 17 Mar 2011 20:45:18 +0000 Subject: [PATCH 052/101] Pep8 --- nova/tests/api/openstack/test_servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 14b72e097f..07ebfdd882 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -548,6 +548,7 @@ class ServersTest(test.TestCase): def test_resized_server_has_correct_status(self): req = self.webreq('/1', 'GET', dict(resize=dict(flavorId=3))) + def fake_migration_get(*args): return {} @@ -556,7 +557,6 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) body = json.loads(res.body) self.assertEqual(body['server']['status'], 'resize-confirm') - def test_confirm_resize_server(self): req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) From 2f4c1802c7e482a447d348f049ff429b3d1a640c Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 18 Mar 2011 16:06:43 -0400 Subject: [PATCH 053/101] fix date formatting in images controller show --- nova/api/openstack/images.py | 6 ++++ nova/tests/api/openstack/fakes.py | 20 +++++++---- nova/tests/api/openstack/test_images.py | 44 ++++++++++++------------- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 98f0dd96b5..94e05823ee 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -143,6 +143,7 @@ class Controller(wsgi.Controller): image = self._service.show(req.environ['nova.context'], image_id) _convert_image_id_to_hash(image) + self._format_image_dates(image) return dict(image=image) def delete(self, req, id): @@ -164,3 +165,8 @@ class Controller(wsgi.Controller): # Users may not modify public images, and that's all that # we support for now. raise faults.Fault(exc.HTTPNotFound()) + + def _format_image_dates(self, image): + for attr in ['created_at', 'updated_at', 'deleted_at']: + if image[attr] is not None: + image[attr] = image[attr].strftime('%Y-%m-%dT%H:%M:%SZ') diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 15f8a5b56a..9573cf1285 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import datetime import json import random @@ -151,22 +152,23 @@ def stub_out_glance(stubs, initial_fixtures=None): for f in self.fixtures] def fake_get_images_detailed(self): - return self.fixtures + return copy.deepcopy(self.fixtures) def fake_get_image_meta(self, image_id): - for f in self.fixtures: - if f['id'] == image_id: - return f + image = self._find_image(image_id) + if image: + return copy.deepcopy(image) raise glance_exc.NotFound def fake_add_image(self, image_meta, data=None): + image_meta = copy.deepcopy(image_meta) id = ''.join(random.choice(string.letters) for _ in range(20)) image_meta['id'] = id self.fixtures.append(image_meta) return image_meta def fake_update_image(self, image_id, image_meta, data=None): - f = self.fake_get_image_meta(image_id) + f = self._find_image(image_id) if not f: raise glance_exc.NotFound @@ -174,7 +176,7 @@ def stub_out_glance(stubs, initial_fixtures=None): return f def fake_delete_image(self, image_id): - f = self.fake_get_image_meta(image_id) + f = self._find_image(image_id) if not f: raise glance_exc.NotFound @@ -183,6 +185,12 @@ def stub_out_glance(stubs, initial_fixtures=None): ##def fake_delete_all(self): ## self.fixtures = [] + def _find_image(self, image_id): + for f in self.fixtures: + if f['id'] == image_id: + return f + return None + GlanceClient = glance_client.Client fake = FakeGlanceClient(initial_fixtures) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 47dd11e5b6..b771966f11 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -21,7 +21,7 @@ and as a WSGI layer """ import json -import datetime +import datetime as dt import shutil import tempfile @@ -177,13 +177,13 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): """Test of the OpenStack API /images application controller""" # Registered images at start of each test. - + now = dt.datetime.utcnow() IMAGE_FIXTURES = [ {'id': '23g2ogk23k4hhkk4k42l', 'imageId': '23g2ogk23k4hhkk4k42l', 'name': 'public image #1', - 'created_at': datetime.datetime.utcnow().isoformat(), - 'updated_at': datetime.datetime.utcnow().isoformat(), + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), 'deleted_at': None, 'deleted': False, 'is_public': True, @@ -192,8 +192,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): {'id': 'slkduhfas73kkaskgdas', 'imageId': 'slkduhfas73kkaskgdas', 'name': 'public image #2', - 'created_at': datetime.datetime.utcnow().isoformat(), - 'updated_at': datetime.datetime.utcnow().isoformat(), + 'created_at': now.isoformat(), + 'updated_at': now.isoformat(), 'deleted_at': None, 'deleted': False, 'is_public': True, @@ -235,20 +235,20 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) - def _is_equivalent_subset(x, y): - if set(x) <= set(y): - for k, v in x.iteritems(): - if x[k] != y[k]: - if x[k] == 'active' and y[k] == 'available': - continue - return False - return True - return False + for image in self.IMAGE_FIXTURES: + expected = { + 'id': abs(hash(image['imageId'])), + 'name': image['name'], + 'status': 'active', + } + self.assertTrue(expected in res_dict['images']) - for image in res_dict['images']: - for image_fixture in self.IMAGE_FIXTURES: - if _is_equivalent_subset(image, image_fixture): - break - else: - self.assertEquals(1, 2, "image %s not in fixtures!" % - str(image)) + def test_show_image(self): + expected = self.IMAGE_FIXTURES[0] + id = abs(hash(expected['id'])) + expected_time = self.now.strftime('%Y-%m-%dT%H:%M:%SZ') + req = webob.Request.blank('/v1.0/images/%s' % id) + res = req.get_response(fakes.wsgi_app()) + actual = json.loads(res.body)['image'] + self.assertEqual(expected_time, actual['created_at']) + self.assertEqual(expected_time, actual['updated_at']) From 9351bd5538ea0fc0a77c4dee13406ac7a71ca1ae Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 18 Mar 2011 17:01:44 -0500 Subject: [PATCH 054/101] Seriously? --- nova/virt/xenapi/vmops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b27fe2216b..4dca26f61c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -383,7 +383,7 @@ class VMOps(object): #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizpng VDI %s for instance %s. Expanding to %sGB") % + LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB") % (vdi_uuid, instance.name, instance.local_gb)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) From 85f50cf496e2c193ddc715f3019b4a4769ab5bd9 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 21 Mar 2011 15:14:24 -0400 Subject: [PATCH 055/101] pep8; various fixes --- nova/api/openstack/servers.py | 1 + nova/api/openstack/views/servers.py | 5 ++--- nova/tests/api/openstack/test_servers.py | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index e3141934be..dafc096ba2 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -513,6 +513,7 @@ class Controller(wsgi.Controller): return kernel_id, ramdisk_id + class ControllerV10(Controller): def _image_id_from_req_data(self, data): return data['server']['imageId'] diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 8d47ac757e..078d5d4841 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -67,9 +67,8 @@ class ViewBuilder(object): # Return the metadata as a dictionary metadata = {} - if 'metadata' in inst: - for item in inst['metadata']: - metadata[item['key']] = item['value'] + for item in inst.get('metadata', []): + metadata[item['key']] = item['value'] inst_dict['metadata'] = metadata inst_dict['hostId'] = '' diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 6e78db9daf..a9e76b2443 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -311,10 +311,19 @@ class ServersTest(test.TestCase): imageRef = 'http://localhost/v1.1/images/2' flavorRef = 'http://localhost/v1.1/flavors/3' - body = dict(server=dict( - name='server_test', imageRef=imageRef, flavorRef=flavorRef, - metadata={'hello': 'world', 'open': 'stack'}, - personality={})) + body = { + 'server': { + 'name': 'server_test', + 'imageRef': imageRef, + 'flavorRef': flavorRef, + 'metadata': { + 'hello': 'world', + 'open': 'stack', + }, + 'personality': {}, + }, + } + req = webob.Request.blank('/v1.1/servers') req.method = 'POST' req.body = json.dumps(body) From 8f7d6b9da89e7154a79ad7d20681d0cb47e042b7 Mon Sep 17 00:00:00 2001 From: Tushar Patil Date: Mon, 21 Mar 2011 12:21:24 -0700 Subject: [PATCH 056/101] Fix for LP Bug #739641 --- smoketests/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/smoketests/base.py b/smoketests/base.py index 3e2446c9a0..31d82b20ba 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -32,7 +32,6 @@ SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) flags.DEFINE_integer('ssh_tries', 3, 'Numer of times to try ssh') -boto_v6 = None class SmokeTestCase(unittest.TestCase): @@ -183,6 +182,9 @@ class SmokeTestCase(unittest.TestCase): TEST_DATA = {} +if FLAGS.use_ipv6: + global boto_v6 + boto_v6 = __import__('boto_v6') class UserSmokeTestCase(SmokeTestCase): From 27ae9700739bd6a1e6f9db90e407f450ff3e770b Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Mon, 21 Mar 2011 16:35:38 -0400 Subject: [PATCH 057/101] added licenses --- nova/tests/image/__init__.py | 17 +++++++++++++++++ nova/tests/image/test_glance.py | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/nova/tests/image/__init__.py b/nova/tests/image/__init__.py index e69de29bb2..fae25bca7e 100644 --- a/nova/tests/image/__init__.py +++ b/nova/tests/image/__init__.py @@ -0,0 +1,17 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 6e94aa9097..fcd686c84f 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -1,3 +1,22 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + import datetime as dt import unittest From 414c615a3ac2e61f312f8383f764114e7d782de1 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Mon, 21 Mar 2011 16:40:26 -0400 Subject: [PATCH 058/101] fix licenses --- nova/tests/image/__init__.py | 3 +-- nova/tests/image/test_glance.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/tests/image/__init__.py b/nova/tests/image/__init__.py index fae25bca7e..b94e2e54ec 100644 --- a/nova/tests/image/__init__.py +++ b/nova/tests/image/__init__.py @@ -1,7 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Openstack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index fcd686c84f..d49b3dfdb5 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -1,7 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Openstack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may From 39783f386a473ed28c786bb72a29e8403503c40c Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Mon, 21 Mar 2011 17:09:53 -0400 Subject: [PATCH 059/101] make bcwaldon happy --- nova/api/openstack/images.py | 2 +- nova/image/glance.py | 6 +++--- nova/tests/api/openstack/test_images.py | 4 ++-- nova/tests/image/test_glance.py | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 94e05823ee..99c14275a7 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -168,5 +168,5 @@ class Controller(wsgi.Controller): def _format_image_dates(self, image): for attr in ['created_at', 'updated_at', 'deleted_at']: - if image[attr] is not None: + if image.get(attr) is not None: image[attr] = image[attr].strftime('%Y-%m-%dT%H:%M:%SZ') diff --git a/nova/image/glance.py b/nova/image/glance.py index fbb578585a..171b28fde2 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -18,7 +18,7 @@ from __future__ import absolute_import -import datetime as dt +import datetime from glance.common import exception as glance_exception @@ -73,7 +73,7 @@ class GlanceImageService(service.BaseImageService): Returns image with known timestamp fields converted to datetime objects """ for attr in ['created_at', 'updated_at', 'deleted_at']: - if attr in image and image[attr] is not None: + if image.get(attr) is not None: image[attr] = self._parse_glance_iso8601_timestamp(image[attr]) return image @@ -81,7 +81,7 @@ class GlanceImageService(service.BaseImageService): """ Parse a subset of iso8601 timestamps into datetime objects """ - return dt.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") + return datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") def show_by_name(self, context, name): """ diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index b771966f11..a866c764d7 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -21,7 +21,7 @@ and as a WSGI layer """ import json -import datetime as dt +import datetime import shutil import tempfile @@ -177,7 +177,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): """Test of the OpenStack API /images application controller""" # Registered images at start of each test. - now = dt.datetime.utcnow() + now = datetime.datetime.utcnow() IMAGE_FIXTURES = [ {'id': '23g2ogk23k4hhkk4k42l', 'imageId': '23g2ogk23k4hhkk4k42l', diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index d49b3dfdb5..30021dbc1c 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -58,12 +58,12 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.service = glance.GlanceImageService(self.client) def test_show_passes_through_to_client(self): - self.client.images = {'xyz': "image"} - self.assertEqual(self.service.show({}, 'xyz'), "image") + self.client.images = {'xyz': {'foo': 'bar'}} + self.assertEqual(self.service.show({}, 'xyz'), {'foo': 'bar'}) def test_detail_passes_through_to_client(self): - self.client.images = {1: "an image"} - self.assertEqual(list(self.service.detail({})), ["an image"]) + self.client.images = {1: {'foo': 'bar'}} + self.assertEqual(list(self.service.detail({})), [{'foo': 'bar'}]) def test_show_makes_create_datetimes(self): create_time = dt.datetime.utcnow() From e1b9db2ac1af8f38084f9794a430e0292f110ed6 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Mon, 21 Mar 2011 17:23:36 -0400 Subject: [PATCH 060/101] get rid of another datetime alias --- nova/tests/image/test_glance.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 30021dbc1c..f1f8504f3d 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -16,7 +16,7 @@ # under the License. -import datetime as dt +import datetime import unittest from nova.image import glance @@ -66,7 +66,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(list(self.service.detail({})), [{'foo': 'bar'}]) def test_show_makes_create_datetimes(self): - create_time = dt.datetime.utcnow() + create_time = datetime.datetime.utcnow() self.client.images = {'xyz': { 'id': "id", 'name': "my awesome image", @@ -76,7 +76,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(actual['created_at'], create_time) def test_show_makes_update_datetimes(self): - update_time = dt.datetime.utcnow() + update_time = datetime.datetime.utcnow() self.client.images = {'abc': { 'id': "id", 'name': "my okay image", @@ -86,7 +86,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(actual['updated_at'], update_time) def test_show_makes_delete_datetimes(self): - delete_time = dt.datetime.utcnow() + delete_time = datetime.datetime.utcnow() self.client.images = {'123': { 'id': "123", 'name': "my lame image", @@ -105,7 +105,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(actual['deleted_at'], None) def test_detail_handles_timestamps(self): - now = dt.datetime.utcnow() + now = datetime.datetime.utcnow() image1 = { 'id': 1, 'name': 'image 1', @@ -126,7 +126,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(i2['deleted_at'], now) def test_get_handles_timestamps(self): - now = dt.datetime.utcnow() + now = datetime.datetime.utcnow() self.client.images = {'abcd': { 'id': 'abcd', 'name': 'nifty image', @@ -144,7 +144,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(actual['deleted_at'], None) def test_create_handles_timestamps(self): - now = dt.datetime.utcnow() + now = datetime.datetime.utcnow() self.client.add_response = { 'id': 'abcd', 'name': 'blah', @@ -166,7 +166,7 @@ class TestGlanceImageServiceDatetimes(unittest.TestCase): self.assertEqual(actual['deleted_at'], None) def test_update_handles_timestamps(self): - now = dt.datetime.utcnow() + now = datetime.datetime.utcnow() self.client.update_response = { 'id': 'abcd', 'name': 'blah', From d1860ce5d26fbbadb2310e8225e924879cde9a6c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 10:35:43 +0100 Subject: [PATCH 061/101] Make synchronized support both external (file based) locks as well as internal (semaphore based) locks. Attempt to make it native thread safe at the expense of never cleaning up semaphores. --- nova/tests/test_misc.py | 34 +++++++++++++++++++-- nova/utils.py | 67 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 1fbaf304f8..961499a60b 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -16,8 +16,12 @@ import errno import os +import random import select +from eventlet import greenpool +from eventlet import greenthread + from nova import test from nova.utils import parse_mailmap, str_dict_replace, synchronized @@ -72,11 +76,37 @@ class LockTestCase(test.TestCase): self.assertEquals(foo.__name__, 'foo', "Wrapped function's name " "got mangled") - def test_synchronized(self): + def test_synchronized_internally(self): + """We can lock across multiple green threads""" + seen_threads = list() + @synchronized('testlock', external=False) + def f(id): + for x in range(10): + seen_threads.append(id) + greenthread.sleep(0) + + threads = [] + pool = greenpool.GreenPool(10) + for i in range(10): + threads.append(pool.spawn(f, i)) + + for thread in threads: + thread.wait() + + self.assertEquals(len(seen_threads), 100) + # Looking at the seen threads, split it into chunks of 10, and verify + # that the last 9 match the first in each chunk. + for i in range(10): + for j in range(9): + self.assertEquals(seen_threads[i*10], seen_threads[i*10+1+j]) + + + def test_synchronized_externally(self): + """We can lock across multiple processes""" rpipe1, wpipe1 = os.pipe() rpipe2, wpipe2 = os.pipe() - @synchronized('testlock') + @synchronized('testlock', external=True) def f(rpipe, wpipe): try: os.write(wpipe, "foo") diff --git a/nova/utils.py b/nova/utils.py index 499af20391..8936614cce 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -41,6 +41,7 @@ from xml.sax import saxutils from eventlet import event from eventlet import greenthread +from eventlet import semaphore from eventlet.green import subprocess None from nova import exception @@ -531,17 +532,69 @@ def loads(s): return json.loads(s) -def synchronized(name): +_semaphores_semaphore = semaphore.Semaphore() +_semaphores = {} + + +class _NoopContextManager(object): + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + +def synchronized(name, external=False): + """Synchronization decorator + + Decorating a method like so: + @synchronized('mylock') + def foo(self, *args): + ... + + ensures that only one thread will execute the bar method at a time. + + Different methods can share the same lock: + @synchronized('mylock') + def foo(self, *args): + ... + + @synchronized('mylock') + def bar(self, *args): + ... + + This way only one of either foo or bar can be executing at a time. + + The external keyword argument denotes whether this lock should work across + multiple processes. This means that if two different workers both run a + a method decorated with @synchronized('mylock', external=True), only one + of them will execute at a time. + """ + def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): - LOG.debug(_("Attempting to grab %(lock)s for method " - "%(method)s..." % {"lock": name, + with _semaphores_semaphore: + if name not in _semaphores: + _semaphores[name] = semaphore.Semaphore() + sem = _semaphores[name] + LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method ' + '"%(method)s"...' % {"lock": name, "method": f.__name__})) - lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, - 'nova-%s.lock' % name)) - with lock: - return f(*args, **kwargs) + with sem: + if external: + LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' + 'method "%(method)s"...' % + {"lock": name, "method": f.__name__})) + lock_file_path = os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name) + lock = lockfile.FileLock(lock_file_path) + else: + lock = _NoopContextManager() + + with lock: + return f(*args, **kwargs) + return inner return wrap From e827b8dbae1faef2cc070c7e26395979571bcd46 Mon Sep 17 00:00:00 2001 From: Hisaharu Ishii Date: Tue, 22 Mar 2011 20:27:51 +0900 Subject: [PATCH 062/101] Wrap update_ra in utils.synchronized. --- nova/network/linux_net.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ee36407a62..e283dee375 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -595,6 +595,7 @@ def update_dhcp(context, network_id): _execute(*command, addl_env=env) +@utils.synchronized('radvd_start') def update_ra(context, network_id): network_ref = db.network_get(context, network_id) From 60a3aa86db1d0e1ea2f680c9587881e45fa99336 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 14:14:47 +0100 Subject: [PATCH 063/101] Make synchronized decorator not leak semaphores, at the expense of not being truly thread safe (but safe enough for Eventlet style green threads). --- nova/network/linux_net.py | 2 +- nova/tests/test_misc.py | 1 - nova/utils.py | 18 +++++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ee36407a62..9bb1685c02 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -274,7 +274,7 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() - @utils.synchronized('iptables') + @utils.synchronized('iptables', external=True) def apply(self): """Apply the current in-memory set of iptables rules diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 961499a60b..c0c72bb124 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -16,7 +16,6 @@ import errno import os -import random import select from eventlet import greenpool diff --git a/nova/utils.py b/nova/utils.py index 8936614cce..c580e805a2 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -574,10 +574,12 @@ def synchronized(name, external=False): def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): - with _semaphores_semaphore: - if name not in _semaphores: - _semaphores[name] = semaphore.Semaphore() - sem = _semaphores[name] + # NOTE(soren): If we ever go natively threaded, this will be racy. + # See http://stackoverflow.com/questions/5390569/dyn\ + # amically-allocating-and-destroying-mutexes + if name not in _semaphores: + _semaphores[name] = semaphore.Semaphore() + sem = _semaphores[name] LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method ' '"%(method)s"...' % {"lock": name, "method": f.__name__})) @@ -593,8 +595,14 @@ def synchronized(name, external=False): lock = _NoopContextManager() with lock: - return f(*args, **kwargs) + retval = f(*args, **kwargs) + # If no-one else is waiting for it, delete it. + # See note about possible raciness above. + if not sem.balance < 1: + del _semaphores[name] + + return retval return inner return wrap From 62f9cc7cee30332143bf4e6e54fd21335db3c8da Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 14:36:32 +0100 Subject: [PATCH 064/101] Convert _cache_image to use utils.synchronized decorator. Disable its test case, since I think it is no longer needed with the tests for synchronized. --- nova/tests/test_virt.py | 2 +- nova/virt/libvirt_conn.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index b214f5ce72..b9cd30a79a 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -62,7 +62,7 @@ class CacheConcurrencyTestCase(test.TestCase): self.stubs.Set(os.path, 'exists', fake_exists) self.stubs.Set(utils, 'execute', fake_execute) - def test_same_fname_concurrency(self): + def notest_same_fname_concurrency(self): """Ensures that the same fname cache runs at a sequentially""" conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e80b9fbdfa..ca8d81f5ff 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -48,7 +48,6 @@ from xml.dom import minidom from eventlet import tpool -from eventlet import semaphore import IPy @@ -552,13 +551,12 @@ class LibvirtConnection(object): os.mkdir(base_dir) base = os.path.join(base_dir, fname) - if fname not in LibvirtConnection._image_sems: - LibvirtConnection._image_sems[fname] = semaphore.Semaphore() - with LibvirtConnection._image_sems[fname]: + @utils.synchronized(fname) + def call_if_not_exists(base, fn, *args, **kwargs): if not os.path.exists(base): fn(target=base, *args, **kwargs) - if not LibvirtConnection._image_sems[fname].locked(): - del LibvirtConnection._image_sems[fname] + + call_if_not_exists(base, fn, *args, **kwargs) if cow: utils.execute('qemu-img', 'create', '-f', 'qcow2', '-o', From 116e5a0b13f3da174137bbff0a5b9b0202caa4a4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 14:37:02 +0100 Subject: [PATCH 065/101] Add --fixes lp: metadata. From 01e7e598d0eb4aab9c3e7f69926a2875cdf22136 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 14:39:35 +0100 Subject: [PATCH 066/101] Get rid of IptablesManager's explicit semaphore. --- nova/network/linux_net.py | 4 ---- nova/virt/libvirt_conn.py | 11 ++++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 9bb1685c02..8cbf8db245 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -21,8 +21,6 @@ import inspect import os import calendar -from eventlet import semaphore - from nova import db from nova import exception from nova import flags @@ -272,8 +270,6 @@ class IptablesManager(object): self.ipv4['nat'].add_chain('floating-snat') self.ipv4['nat'].add_rule('snat', '-j $floating-snat') - self.semaphore = semaphore.Semaphore() - @utils.synchronized('iptables', external=True) def apply(self): """Apply the current in-memory set of iptables rules diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index ca8d81f5ff..902866167b 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1766,14 +1766,11 @@ class IptablesFirewallDriver(FirewallDriver): def refresh_security_group_members(self, security_group): pass + @utils.synchronized('iptables', external=True) def refresh_security_group_rules(self, security_group): - # We use the semaphore to make sure noone applies the rule set - # after we've yanked the existing rules but before we've put in - # the new ones. - with self.iptables.semaphore: - for instance in self.instances.values(): - self.remove_filters_for_instance(instance) - self.add_filters_for_instance(instance) + for instance in self.instances.values(): + self.remove_filters_for_instance(instance) + self.add_filters_for_instance(instance) self.iptables.apply() def _security_group_chain_name(self, security_group_id): From 804083b6ba811834c0bf9d5e2edcdf0130d7d1ce Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 14:50:53 +0100 Subject: [PATCH 067/101] IptablesManager.semaphore is no more. --- nova/network/linux_net.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 8cbf8db245..9faa7de07b 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -277,28 +277,23 @@ class IptablesManager(object): This will blow away any rules left over from previous runs of the same component of Nova, and replace them with our current set of rules. This happens atomically, thanks to iptables-restore. - - We wrap the call in a semaphore lock, so that we don't race with - ourselves. In the event of a race with another component running - an iptables-* command at the same time, we retry up to 5 times. """ - with self.semaphore: - s = [('iptables', self.ipv4)] - if FLAGS.use_ipv6: - s += [('ip6tables', self.ipv6)] + s = [('iptables', self.ipv4)] + if FLAGS.use_ipv6: + s += [('ip6tables', self.ipv6)] - for cmd, tables in s: - for table in tables: - current_table, _ = self.execute('sudo', - '%s-save' % (cmd,), - '-t', '%s' % (table,), - attempts=5) - current_lines = current_table.split('\n') - new_filter = self._modify_rules(current_lines, - tables[table]) - self.execute('sudo', '%s-restore' % (cmd,), - process_input='\n'.join(new_filter), - attempts=5) + for cmd, tables in s: + for table in tables: + current_table, _ = self.execute('sudo', + '%s-save' % (cmd,), + '-t', '%s' % (table,), + attempts=5) + current_lines = current_table.split('\n') + new_filter = self._modify_rules(current_lines, + tables[table]) + self.execute('sudo', '%s-restore' % (cmd,), + process_input='\n'.join(new_filter), + attempts=5) def _modify_rules(self, current_lines, table, binary=None): unwrapped_chains = table.unwrapped_chains From de2ecf115ff0baf43fa530807997513c728ffdaf Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 15:16:08 +0100 Subject: [PATCH 068/101] Fix locking problem in security group refresh code. --- nova/virt/libvirt_conn.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 902866167b..fcd78b3b20 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1766,12 +1766,15 @@ class IptablesFirewallDriver(FirewallDriver): def refresh_security_group_members(self, security_group): pass - @utils.synchronized('iptables', external=True) def refresh_security_group_rules(self, security_group): + self.do_refresh_security_group_rules(security_group) + self.iptables.apply() + + @utils.synchronized('iptables', external=True) + def do_refresh_security_group_rules(self, security_group): for instance in self.instances.values(): self.remove_filters_for_instance(instance) self.add_filters_for_instance(instance) - self.iptables.apply() def _security_group_chain_name(self, security_group_id): return 'nova-sg-%s' % (security_group_id,) From b718ec133a5ed64abca2c01232986bc3dfeae3c0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 15:50:34 +0100 Subject: [PATCH 069/101] Add correct bug fixing metadata. From d2494199df440809bbfbc55868b0dd57053868ed Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 16:23:47 +0100 Subject: [PATCH 070/101] Remove checks in _cache_image tests that were too implementation specific. --- nova/tests/test_virt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index b9cd30a79a..6bafac39f1 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -62,7 +62,7 @@ class CacheConcurrencyTestCase(test.TestCase): self.stubs.Set(os.path, 'exists', fake_exists) self.stubs.Set(utils, 'execute', fake_execute) - def notest_same_fname_concurrency(self): + def test_same_fname_concurrency(self): """Ensures that the same fname cache runs at a sequentially""" conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() @@ -77,13 +77,11 @@ class CacheConcurrencyTestCase(test.TestCase): eventlet.sleep(0) try: self.assertFalse(done2.ready()) - self.assertTrue('fname' in conn._image_sems) finally: wait1.send() done1.wait() eventlet.sleep(0) self.assertTrue(done2.ready()) - self.assertFalse('fname' in conn._image_sems) def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" From 9aac55b650e9f39c5771d4683e51af5eac6204bb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 16:24:03 +0100 Subject: [PATCH 071/101] Add a test for leaked semaphores. --- nova/tests/test_misc.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index c0c72bb124..8fc5d67c07 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -22,7 +22,8 @@ from eventlet import greenpool from eventlet import greenthread from nova import test -from nova.utils import parse_mailmap, str_dict_replace, synchronized +from nova import utils +from nova.utils import parse_mailmap, str_dict_replace class ProjectTestCase(test.TestCase): @@ -66,7 +67,7 @@ class ProjectTestCase(test.TestCase): class LockTestCase(test.TestCase): def test_synchronized_wrapped_function_metadata(self): - @synchronized('whatever') + @utils.synchronized('whatever') def foo(): """Bar""" pass @@ -77,8 +78,9 @@ class LockTestCase(test.TestCase): def test_synchronized_internally(self): """We can lock across multiple green threads""" + saved_sem_num = len(utils._semaphores) seen_threads = list() - @synchronized('testlock', external=False) + @utils.synchronized('testlock2', external=False) def f(id): for x in range(10): seen_threads.append(id) @@ -99,13 +101,15 @@ class LockTestCase(test.TestCase): for j in range(9): self.assertEquals(seen_threads[i*10], seen_threads[i*10+1+j]) + self.assertEqual(saved_sem_num, len(utils._semaphores), + "Semaphore leak detected") def test_synchronized_externally(self): """We can lock across multiple processes""" rpipe1, wpipe1 = os.pipe() rpipe2, wpipe2 = os.pipe() - @synchronized('testlock', external=True) + @utils.synchronized('testlock1', external=True) def f(rpipe, wpipe): try: os.write(wpipe, "foo") From b2bdeb82024b1a015ccb2ad14606d6e9ccf80aa8 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 16:29:37 +0100 Subject: [PATCH 072/101] pep8 --- nova/tests/test_misc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 8fc5d67c07..4e17e1ce07 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -80,6 +80,7 @@ class LockTestCase(test.TestCase): """We can lock across multiple green threads""" saved_sem_num = len(utils._semaphores) seen_threads = list() + @utils.synchronized('testlock2', external=False) def f(id): for x in range(10): @@ -99,7 +100,8 @@ class LockTestCase(test.TestCase): # that the last 9 match the first in each chunk. for i in range(10): for j in range(9): - self.assertEquals(seen_threads[i*10], seen_threads[i*10+1+j]) + self.assertEquals(seen_threads[i * 10], + seen_threads[i * 10 + 1 + j]) self.assertEqual(saved_sem_num, len(utils._semaphores), "Semaphore leak detected") From 4e33ab9fc16d580fbcf57da8e6e2228ad27cc1af Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Tue, 22 Mar 2011 11:46:00 -0400 Subject: [PATCH 073/101] Adding more docstrings. image_id and instance_type fields of an instance will always exist, so no reason to check if keys exist. --- nova/api/openstack/__init__.py | 4 ++++ nova/api/openstack/views/servers.py | 25 ++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 35b04f8633..21d354f1c7 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -131,6 +131,8 @@ class APIRouter(wsgi.Router): class APIRouterV10(APIRouter): + ''' Defines routes specific to OpenStack API V1.0 ''' + def _setup_routes(self, mapper): APIRouter._setup_routes(self, mapper) mapper.resource("server", "servers", @@ -140,6 +142,8 @@ class APIRouterV10(APIRouter): class APIRouterV11(APIRouter): + ''' Defines routes specific to OpenStack API V1.1 ''' + def _setup_routes(self, mapper): APIRouter._setup_routes(self, mapper) mapper.resource("server", "servers", diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 078d5d4841..3100c46b5e 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -34,19 +34,18 @@ class ViewBuilder(object): self.addresses_builder = addresses_builder def build(self, inst, is_detail): - """ - Coerces into dictionary format, mapping everything to - Rackspace-like attributes for return - """ + ''' Returns a dict that represenst a server ''' if is_detail: return self._build_detail(inst) else: return self._build_simple(inst) def _build_simple(self, inst): - return dict(server=dict(id=inst['id'], name=inst['display_name'])) + ''' Returns a simple model of a server ''' + return dict(server=dict(id=inst['id'], name=inst['display_name'])) def _build_detail(self, inst): + ''' Returns a detailed model of a server ''' power_mapping = { None: 'build', power_state.NOSTATE: 'build', @@ -81,36 +80,36 @@ class ViewBuilder(object): return dict(server=inst_dict) def _build_image(self, response, inst): + ''' Returns the image sub-resource of a server ''' raise NotImplementedError() def _build_flavor(self, response, inst): + ''' Returns the flavor sub-resource of a server ''' raise NotImplementedError() class ViewBuilderV10(ViewBuilder): + ''' Models an Openstack API V1.0 server response ''' + def _build_image(self, response, inst): - if inst.get('image_id') != None: - response['imageId'] = inst['image_id'] + response['imageId'] = inst['image_id'] def _build_flavor(self, response, inst): - if inst.get('instance_type') != None: - response['flavorId'] = inst['instance_type'] + response['flavorId'] = inst['instance_type'] class ViewBuilderV11(ViewBuilder): + ''' Models an Openstack API V1.0 server response ''' + def __init__(self, addresses_builder, flavor_builder, image_builder): ViewBuilder.__init__(self, addresses_builder) self.flavor_builder = flavor_builder self.image_builder = image_builder def _build_image(self, response, inst): - if inst.get('image_id') == None: - return image_id = inst["image_id"] response["imageRef"] = self.image_builder.generate_href(image_id) def _build_flavor(self, response, inst): - if inst.get('instance_type') == None: - return flavor_id = inst["instance_type"] response["flavorRef"] = self.flavor_builder.generate_href(flavor_id) From 06815cb729d8687403fc736ae6125c26867f42b3 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Mar 2011 17:13:48 +0100 Subject: [PATCH 074/101] Remove unused global semaphore. --- nova/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index c580e805a2..8b9ce47340 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -532,7 +532,6 @@ def loads(s): return json.loads(s) -_semaphores_semaphore = semaphore.Semaphore() _semaphores = {} From e648698bd171357228881a10d76e7853938e8feb Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 22 Mar 2011 17:00:36 +0000 Subject: [PATCH 075/101] Fix --- nova/tests/test_localization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_localization.py b/nova/tests/test_localization.py index 393d71038a..132a308fd7 100644 --- a/nova/tests/test_localization.py +++ b/nova/tests/test_localization.py @@ -21,9 +21,9 @@ import sys import unittest import nova +from nova import test - -class LocalizationTestCase(unittest.TestCase): +class LocalizationTestCase(test.TestCase): def test_multiple_positional_format_placeholders(self): pat = re.compile("\W_\(") single_pat = re.compile("\W%\W") From 493e87976b7eb273f4115d46c91ad73671abb796 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Tue, 22 Mar 2011 13:18:08 -0400 Subject: [PATCH 076/101] Now using urlparse to parse a url to grab id out of it. --- nova/api/openstack/common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index b224cbfb45..99fba8fef3 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -17,6 +17,7 @@ import re from nova import exception +from urlparse import urlparse from webob import exc import webob.exc @@ -78,7 +79,7 @@ def get_image_id_from_image_hash(image_service, context, image_hash): def get_id_from_href(href): - m = re.match(r'http.+/.+/(\d)+$', href) - if not m: + try: + return int(urlparse(href).path.split('/')[-1]) + except: raise exc.HTTPBadRequest(_('could not parse id from href')) - return int(m.group(1)) From 116c0d52d21ebd6ed55a61467aac5d8c06a4b086 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 22 Mar 2011 17:46:17 +0000 Subject: [PATCH 077/101] Merge stuff --- nova/api/openstack/servers.py | 4 ++-- nova/api/openstack/views/servers.py | 5 +++-- nova/compute/api.py | 8 ++++---- nova/db/sqlalchemy/api.py | 4 ++-- nova/virt/xenapi/vmops.py | 4 ++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index db5942e92b..f3367e118a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -36,7 +36,7 @@ from nova.api.openstack.views import addresses as addresses_views from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state -prom nova.quota import QuotaError +from nova.quota import QuotaError import nova.api.openstack @@ -44,7 +44,7 @@ LOG = logging.getLogger('server') FLAGS = flags.FLAGS -plass Controller(wsgi.Controller): +class Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ _serialization_metadata = { diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 6d54a7a7ec..9fd25999ab 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -89,8 +89,9 @@ class ViewBuilder(object): migration = db.migration_get_by_instance_and_status(ctxt, inst['id'], 'finished') inst_dict['status'] = 'resize-confirm' - except Exception, e: - inst_dict['status'] = power_mapping[inst_dict['status']] + except: + pass + inst_dict['addresses'] = self.addresses_builder.build(inst) # Return the metadata as a dictionary diff --git a/nova/compute/api.py b/nova/compute/api.py index dbf99e7c5b..748aba004e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -489,15 +489,15 @@ class API(base.Base): def resize(self, context, instance_id, flavor_id): """Resize a running instance.""" instance = self.db.instance_get(context, instance_id) - LOG.debug(_("Resizing instance %(instance_type['name'] to flavor" - "%(flavor_id)") % locals()) current_instance_type = self.db.instance_type_get_by_name( context, instance['instance_type']) new_instance_type = self.db.instance_type_get_by_flavor_id( context, flavor_id) - LOG.debug(_("Old instance type %s -> New instance type %s"), - (current_instance_type['name'], new_instance_type['name'])) + current_instance_type_name = current_instance_type['name'] + new_instance_type_name = new_instance_type['name'] + LOG.debug(_("Old instance type %(current_instance_type_name)s, " + " new instance type %(new_instance_type_name)s") % locals()) if not new_instance_type: raise exception.ApiError(_("Requested flavor does not exist")) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index ac7f7cbf10..98810cb483 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2220,8 +2220,8 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(instance_id=instance_id).\ filter_by(status=status).first() if not result: - raise exception.NotFound(_("No migration found for instance %s" - "with status %s" % (instance_id, status))) + raise exception.NotFound(_("No migration found for instance " + "%(instance_id) with status %(status)") % locals()) return result diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 1f5d2d155a..c1a65c997c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -387,8 +387,8 @@ class VMOps(object): #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB"), - (vdi_uuid, instance.name, instance.local_gb)) + LOG.debug(_("Resizing VDI %(vdi_uuid) for instance %(instance.name). " + "Expanding to %(instance.local_gb)GB") % locals()) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) LOG.debug(_("Resize instance %s complete") % (instance.name)) From 3b3889a19c4efa8dc917f772613543780f361df3 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 22 Mar 2011 17:55:40 +0000 Subject: [PATCH 078/101] tweak --- nova/virt/xenapi/vmops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c1a65c997c..8ac944966b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -387,8 +387,8 @@ class VMOps(object): #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizing VDI %(vdi_uuid) for instance %(instance.name). " - "Expanding to %(instance.local_gb)GB") % locals()) + LOG.debug(_("Resizing VDI %(vdi_uuid) for instance %(instance.name)s. " + "Expanding to %(instance.local_gb)f GB") % locals()) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) LOG.debug(_("Resize instance %s complete") % (instance.name)) From 4c76bcc12954734d19afcb5e4519e35c23e39d6d Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 22 Mar 2011 18:04:09 +0000 Subject: [PATCH 079/101] tweak --- nova/virt/xenapi/vmops.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 8ac944966b..383096d63a 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -387,8 +387,10 @@ class VMOps(object): #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizing VDI %(vdi_uuid) for instance %(instance.name)s. " - "Expanding to %(instance.local_gb)f GB") % locals()) + instance_name = instance.name + instance_local_gb = instance.local_gb + LOG.debug(_("Resizing VDI %(vdi_uuid)s for instance %(instance_name)s." + " Expanding to %(instance_local_gb)d GB") % locals()) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) LOG.debug(_("Resize instance %s complete") % (instance.name)) From 8792383dfbd630388e6a51a76910e73203a3793f Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 22 Mar 2011 18:24:00 +0000 Subject: [PATCH 080/101] Tweak --- nova/api/openstack/views/servers.py | 4 ++++ .../migrate_repo/versions/012_add_flavors_to_migrations.py | 1 + nova/tests/test_localization.py | 1 + 3 files changed, 6 insertions(+) diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 9fd25999ab..709052f222 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -16,7 +16,10 @@ # under the License. import hashlib + from nova.compute import power_state +import nova.context +from nova import db from nova.api.openstack import common from nova.api.openstack.views import addresses as addresses_view from nova.api.openstack.views import flavors as flavors_view @@ -86,6 +89,7 @@ class ViewBuilder(object): inst_dict['status'] = power_mapping[inst_dict['status']] try: + ctxt = nova.context.get_admin_context() migration = db.migration_get_by_instance_and_status(ctxt, inst['id'], 'finished') inst_dict['status'] = 'resize-confirm' diff --git a/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py index e677ba14de..3fb92e85c1 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py @@ -43,6 +43,7 @@ def upgrade(migrate_engine): migrations.create_column(old_flavor_id) migrations.create_column(new_flavor_id) + def downgrade(migrate_engine): meta.bind = migrate_engine migrations.drop_column(old_flavor_id) diff --git a/nova/tests/test_localization.py b/nova/tests/test_localization.py index 132a308fd7..a25809a791 100644 --- a/nova/tests/test_localization.py +++ b/nova/tests/test_localization.py @@ -23,6 +23,7 @@ import unittest import nova from nova import test + class LocalizationTestCase(test.TestCase): def test_multiple_positional_format_placeholders(self): pat = re.compile("\W_\(") From ca37b31d64f9c5cf32ca7e6015176ef36e702dce Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Tue, 22 Mar 2011 16:04:27 -0400 Subject: [PATCH 081/101] Updating doc strings in accordance with PEP 257. Fixing order of imports in common.py. --- nova/api/openstack/__init__.py | 4 ++-- nova/api/openstack/common.py | 16 +++++++++++----- nova/api/openstack/views/servers.py | 22 ++++++++++++---------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 21d354f1c7..5f96482109 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -131,7 +131,7 @@ class APIRouter(wsgi.Router): class APIRouterV10(APIRouter): - ''' Defines routes specific to OpenStack API V1.0 ''' + """Define routes specific to OpenStack API V1.0.""" def _setup_routes(self, mapper): APIRouter._setup_routes(self, mapper) @@ -142,7 +142,7 @@ class APIRouterV10(APIRouter): class APIRouterV11(APIRouter): - ''' Defines routes specific to OpenStack API V1.1 ''' + """Define routes specific to OpenStack API V1.1.""" def _setup_routes(self, mapper): APIRouter._setup_routes(self, mapper) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 99fba8fef3..21ceec45e2 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -15,11 +15,11 @@ # License for the specific language governing permissions and limitations # under the License. -import re -from nova import exception from urlparse import urlparse -from webob import exc -import webob.exc + +import webob + +from nova import exception def limited(items, request, max_limit=1000): @@ -79,7 +79,13 @@ def get_image_id_from_image_hash(image_service, context, image_hash): def get_id_from_href(href): + """Return the id portion of a url. + + Given: http://www.foo.com/bar/123?q=4 + Returns: 4 + + """ try: return int(urlparse(href).path.split('/')[-1]) except: - raise exc.HTTPBadRequest(_('could not parse id from href')) + raise webob.exc.HTTPBadRequest(_('could not parse id from href')) diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 3100c46b5e..fad361bd42 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -25,27 +25,29 @@ from nova import utils class ViewBuilder(object): - ''' - Models a server response as a python dictionary. + """Model a server response as a python dictionary. + + Public methods: build Abstract methods: _build_image, _build_flavor - ''' + + """ def __init__(self, addresses_builder): self.addresses_builder = addresses_builder def build(self, inst, is_detail): - ''' Returns a dict that represenst a server ''' + """Return a dict that represenst a server.""" if is_detail: return self._build_detail(inst) else: return self._build_simple(inst) def _build_simple(self, inst): - ''' Returns a simple model of a server ''' + """Return a simple model of a server.""" return dict(server=dict(id=inst['id'], name=inst['display_name'])) def _build_detail(self, inst): - ''' Returns a detailed model of a server ''' + """Returns a detailed model of a server.""" power_mapping = { None: 'build', power_state.NOSTATE: 'build', @@ -80,16 +82,16 @@ class ViewBuilder(object): return dict(server=inst_dict) def _build_image(self, response, inst): - ''' Returns the image sub-resource of a server ''' + """Return the image sub-resource of a server.""" raise NotImplementedError() def _build_flavor(self, response, inst): - ''' Returns the flavor sub-resource of a server ''' + """Return the flavor sub-resource of a server.""" raise NotImplementedError() class ViewBuilderV10(ViewBuilder): - ''' Models an Openstack API V1.0 server response ''' + """Model an Openstack API V1.0 server response.""" def _build_image(self, response, inst): response['imageId'] = inst['image_id'] @@ -99,7 +101,7 @@ class ViewBuilderV10(ViewBuilder): class ViewBuilderV11(ViewBuilder): - ''' Models an Openstack API V1.0 server response ''' + """Model an Openstack API V1.0 server response.""" def __init__(self, addresses_builder, flavor_builder, image_builder): ViewBuilder.__init__(self, addresses_builder) From 3796b5a8fc2baa9a35ebbc721735f22e952e6aa3 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 23 Mar 2011 00:31:50 -0400 Subject: [PATCH 082/101] Fix some crypto strangeness (\n in file_name field of certificates, wrong IMPL method for certificate_update). --- nova/crypto.py | 3 ++- nova/db/api.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index 2a8d4abca6..b112e5b924 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -26,6 +26,7 @@ import gettext import hashlib import os import shutil +import string import struct import tempfile import time @@ -267,7 +268,7 @@ def _sign_csr(csr_text, ca_folder): './openssl.cnf', '-infiles', inbound) out, _err = utils.execute('openssl', 'x509', '-in', outbound, '-serial', '-noout') - serial = out.rpartition("=")[2] + serial = string.strip(out.rpartition("=")[2]) os.chdir(start) with open(outbound, "r") as crtfile: return (serial, crtfile.read()) diff --git a/nova/db/api.py b/nova/db/api.py index add5bd83e5..afc1bff2f7 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -214,7 +214,7 @@ def certificate_update(context, certificate_id, values): Raises NotFound if service does not exist. """ - return IMPL.service_update(context, certificate_id, values) + return IMPL.certificate_update(context, certificate_id, values) ################### From 846b09925da07c2858052143d5fff4766a782cf1 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Tue, 22 Mar 2011 22:54:34 -0700 Subject: [PATCH 083/101] Fix for lp740742 - format describe_instance_output correctly to prevent errors in dashboard --- nova/api/ec2/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index 037184b40f..d8d90ad833 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -120,7 +120,8 @@ class AdminController(object): def describe_instance_types(self, context, **_kwargs): """Returns all active instance types data (vcpus, memory, etc.)""" - return {'instanceTypeSet': [db.instance_type_get_all(context)]} + return {'instanceTypeSet': [instance_dict(v) for v in + db.instance_type_get_all(context).values()]} def describe_user(self, _context, name, **_kwargs): """Returns user data, including access and secret keys.""" From a822941d1fbfcfff7d52e2e42f2a50cb8aca6f0d Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 01:02:13 -0700 Subject: [PATCH 084/101] Report the exception (happens when can't import libvirt) --- nova/compute/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 576937cd86..4f338135be 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -118,8 +118,8 @@ class ComputeManager(manager.Manager): try: self.driver = utils.import_object(compute_driver) - except ImportError: - LOG.error("Unable to load the virtualization driver.") + except ImportError as e: + LOG.error(_("Unable to load the virtualization driver: %s") % (e)) sys.exit(1) self.network_manager = utils.import_object(FLAGS.network_manager) From 3362be7e9f2feda33e14ab4fb7c6f70277df1cf5 Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Wed, 23 Mar 2011 12:53:10 +0000 Subject: [PATCH 085/101] Checking whether cidr_v6 is not null before populating ipv6 key in network info map (VMOps._get_network_info) --- nova/virt/xenapi/vmops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 499c6d8a1b..b51489ebc1 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -723,8 +723,9 @@ class VMOps(object): 'mac': instance.mac_address, 'rxtx_cap': flavor['rxtx_cap'], 'dns': [network['dns']], - 'ips': [ip_dict(ip) for ip in network_IPs], - 'ip6s': [ip6_dict(ip) for ip in network_IPs]} + 'ips': [ip_dict(ip) for ip in network_IPs]} + if network['cidr_v6']: + info['ip6s'] = [ip6_dict(ip) for ip in network_IPs] network_info.append((network, info)) return network_info From ea92a88b727814698dbc4ebf5dc705677d636445 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Wed, 23 Mar 2011 14:05:21 -0400 Subject: [PATCH 086/101] Using super to call parent _setup_routes in APIRouter subclasses. --- nova/api/openstack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 5f96482109..e68110bc4b 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -134,7 +134,7 @@ class APIRouterV10(APIRouter): """Define routes specific to OpenStack API V1.0.""" def _setup_routes(self, mapper): - APIRouter._setup_routes(self, mapper) + super(APIRouterV10, self)._setup_routes(mapper) mapper.resource("server", "servers", controller=servers.ControllerV10(), collection={'detail': 'GET'}, @@ -145,7 +145,7 @@ class APIRouterV11(APIRouter): """Define routes specific to OpenStack API V1.1.""" def _setup_routes(self, mapper): - APIRouter._setup_routes(self, mapper) + super(APIRouterV11, self)._setup_routes(mapper) mapper.resource("server", "servers", controller=servers.ControllerV11(), collection={'detail': 'GET'}, From c3d47689a762bfa4aa38c7d4700bb1969d37d1d1 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 23 Mar 2011 18:56:23 +0000 Subject: [PATCH 087/101] merge prop changes --- nova/api/openstack/views/servers.py | 9 +++------ nova/compute/api.py | 13 ++++++++++++- nova/compute/manager.py | 8 ++++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 709052f222..a21a6e7ff6 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -18,6 +18,7 @@ import hashlib from nova.compute import power_state +import nova.compute.api import nova.context from nova import db from nova.api.openstack import common @@ -87,14 +88,10 @@ class ViewBuilder(object): for k, v in mapped_keys.iteritems(): inst_dict[k] = inst[v] + ctxt = nova.context.get_admin_context() inst_dict['status'] = power_mapping[inst_dict['status']] - try: - ctxt = nova.context.get_admin_context() - migration = db.migration_get_by_instance_and_status(ctxt, - inst['id'], 'finished') + if nova.compute.api.has_finished_migration(ctxt, inst['id']): inst_dict['status'] = 'resize-confirm' - except: - pass inst_dict['addresses'] = self.addresses_builder.build(inst) diff --git a/nova/compute/api.py b/nova/compute/api.py index 748aba004e..78110c048e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -253,6 +253,16 @@ class API(base.Base): return [dict(x.iteritems()) for x in instances] + def has_finished_migration(self, context, instance_id): + """Retrieves whether or not a finished migration exists for + an instance""" + try: + db.migration_get_by_instance_and_status(ctxt, inst['id'], + 'finished') + return True + except Exception, e: + return False + def ensure_default_security_group(self, context): """ Create security group for the security context if it does not already exist @@ -499,7 +509,8 @@ class API(base.Base): LOG.debug(_("Old instance type %(current_instance_type_name)s, " " new instance type %(new_instance_type_name)s") % locals()) if not new_instance_type: - raise exception.ApiError(_("Requested flavor does not exist")) + raise exception.ApiError(_("Requested flavor %(flavor_id)d " + "does not exist") % locals()) if current_instance_type['memory_mb'] >= \ new_instance_type['memory_mb']: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 78ef33ac25..ac63f68ea4 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -458,8 +458,8 @@ class ComputeManager(manager.Manager): instance_type = self.db.instance_type_get_by_flavor_id(context, migration_ref['old_flavor_id']) - #Just roll back the record. There's no need to resize down since - #the 'old' VM already has the preferred attributes + # Just roll back the record. There's no need to resize down since + # the 'old' VM already has the preferred attributes self.db.instance_update(context, instance_id, dict(memory_mb=instance_type['memory_mb'], vcpus=instance_type['vcpus'], @@ -536,8 +536,8 @@ class ComputeManager(manager.Manager): migration_ref = self.db.migration_get(context, migration_id) instance_ref = self.db.instance_get(context, migration_ref['instance_id']) - #TODO(mdietz): apply the rest of the instance_type attributes going - #after they're supported + # TODO(mdietz): apply the rest of the instance_type attributes going + # after they're supported instance_type = self.db.instance_type_get_by_flavor_id(context, migration_ref['new_flavor_id']) self.db.instance_update(context, instance_id, From 1aa576ee43cdf6520df6b5c8429f8d426bafc72a Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 23 Mar 2011 18:59:24 +0000 Subject: [PATCH 088/101] Moving the migration yet again --- ..._flavors_to_migrations.py => 013_add_flavors_to_migrations.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nova/db/sqlalchemy/migrate_repo/versions/{012_add_flavors_to_migrations.py => 013_add_flavors_to_migrations.py} (100%) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/013_add_flavors_to_migrations.py similarity index 100% rename from nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py rename to nova/db/sqlalchemy/migrate_repo/versions/013_add_flavors_to_migrations.py From 5a5c7d22e7a00c9a3b34f8c08db70b644eee2d92 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 23 Mar 2011 19:16:03 +0000 Subject: [PATCH 089/101] Unit test cleanup --- nova/api/openstack/views/servers.py | 5 +++-- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index a21a6e7ff6..18d31a29d9 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -18,7 +18,7 @@ import hashlib from nova.compute import power_state -import nova.compute.api +import nova.compute import nova.context from nova import db from nova.api.openstack import common @@ -90,7 +90,8 @@ class ViewBuilder(object): ctxt = nova.context.get_admin_context() inst_dict['status'] = power_mapping[inst_dict['status']] - if nova.compute.api.has_finished_migration(ctxt, inst['id']): + compute_api = nova.compute.API() + if compute_api.has_finished_migration(ctxt, inst['id']): inst_dict['status'] = 'resize-confirm' inst_dict['addresses'] = self.addresses_builder.build(inst) diff --git a/nova/compute/api.py b/nova/compute/api.py index 78110c048e..c2738f6f5f 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -257,7 +257,7 @@ class API(base.Base): """Retrieves whether or not a finished migration exists for an instance""" try: - db.migration_get_by_instance_and_status(ctxt, inst['id'], + db.migration_get_by_instance_and_status(context, instance_id, 'finished') return True except Exception, e: diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 98810cb483..d7b5aff462 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2221,7 +2221,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found for instance " - "%(instance_id) with status %(status)") % locals()) + "%(instance_id)s with status %(status)s") % locals()) return result From abb764f51385a0b811b23379d78f7db027d4cca5 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 14:41:35 -0500 Subject: [PATCH 090/101] Automatically unrescue instances after a given timeout --- nova/compute/manager.py | 12 ++++- nova/utils.py | 7 +++ nova/virt/libvirt_conn.py | 4 ++ nova/virt/xenapi/vmops.py | 95 +++++++++++++++++++++++++++++---------- nova/virt/xenapi_conn.py | 4 ++ 5 files changed, 96 insertions(+), 26 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 576937cd86..3834c33ab1 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -65,8 +65,11 @@ flags.DEFINE_string('console_host', socket.gethostname(), 'Console proxy host to use to connect to instances on' 'this host.') flags.DEFINE_integer('live_migration_retry_count', 30, - ("Retry count needed in live_migration." - " sleep 1 sec for each count")) + "Retry count needed in live_migration." + " sleep 1 sec for each count") +flags.DEFINE_integer("rescue_timeout", 0, + "Automatically unrescue an instance after N hours." + " Set to 0 to disable.") LOG = logging.getLogger('nova.compute.manager') @@ -132,6 +135,11 @@ class ComputeManager(manager.Manager): """ self.driver.init_host(host=self.host) + def periodic_tasks(self, context=None): + """Tasks to be run at a periodic interval.""" + super(ComputeManager, self).periodic_tasks(context) + self.driver.poll_rescued_instances(FLAGS.rescue_timeout) + def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? diff --git a/nova/utils.py b/nova/utils.py index 499af20391..38cdb80219 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -334,6 +334,13 @@ def utcnow(): utcnow.override_time = None +def is_then_greater(then, seconds): + if utcnow() - then > datetime.timedelta(seconds=seconds): + return True + else: + return False + + def utcnow_ts(): """Timestamp version of our utcnow function.""" return time.mktime(utcnow().timetuple()) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e80b9fbdfa..07545382df 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -412,6 +412,10 @@ class LibvirtConnection(object): # the normal xml file, we can just call reboot here self.reboot(instance) + @exception.wrap_exception + def poll_rescued_instances(self, timeout): + pass + @exception.wrap_exception def spawn(self, instance): xml = self.to_xml(instance) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 61ff00903e..f46ac3b7ed 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -51,6 +51,7 @@ class VMOps(object): def __init__(self, session): self.XenAPI = session.get_imported_xenapi() self._session = session + self.poll_rescue_last_ran = None VMHelper.XenAPI = self.XenAPI @@ -462,6 +463,10 @@ class VMOps(object): except self.XenAPI.Failure, exc: LOG.exception(exc) + def _shutdown_rescue(self, vm_ref): + """Shutdown a rescue instance""" + self._session.call_xenapi("Async.VM.hard_shutdown", rescue_vm_ref) + def _destroy_vdis(self, instance, vm_ref): """Destroys all VDIs associated with a VM""" instance_id = instance.id @@ -479,6 +484,24 @@ class VMOps(object): except self.XenAPI.Failure, exc: LOG.exception(exc) + def _destroy_rescue_vdis(self, rescue_vm_ref): + """Destroys all VDIs associated with a rescued VM""" + vdi_refs = VMHelper.lookup_vm_vdis(self._session, rescue_vm_ref) + for vdi_ref in vdi_refs: + try: + self._session.call_xenapi("Async.VDI.destroy", vdi_ref) + except self.XenAPI.Failure: + continue + + def _destroy_rescue_vbds(self, rescue_vm_ref): + """Destroys all VBDs tied to a rescue VM""" + vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) + for vbd_ref in vbd_refs: + _vbd_ref = self._session.get_xenapi().VBD.get_record(vbd_ref) + if _vbd_ref["userdevice"] == "1": + VMHelper.unplug_vbd(self._session, vbd_ref) + VMHelper.destroy_vbd(self._session, vbd_ref) + def _destroy_kernel_ramdisk(self, instance, vm_ref): """ Three situations can occur: @@ -529,6 +552,10 @@ class VMOps(object): LOG.debug(_("Instance %(instance_id)s VM destroyed") % locals()) + def _destroy_rescue(self, vm_ref): + """Destroy a rescue instance""" + self._session.call_xenapi("Async.VM.destroy", rescue_vm_ref) + def destroy(self, instance): """ Destroy VM instance @@ -632,41 +659,61 @@ class VMOps(object): """ rescue_vm_ref = VMHelper.lookup(self._session, - instance.name + "-rescue") + instance.name + "-rescue") if not rescue_vm_ref: raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) original_vm_ref = self._get_vm_opaque_ref(instance) - vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) - instance._rescue = False - for vbd_ref in vbd_refs: - _vbd_ref = self._session.get_xenapi().VBD.get_record(vbd_ref) - if _vbd_ref["userdevice"] == "1": - VMHelper.unplug_vbd(self._session, vbd_ref) - VMHelper.destroy_vbd(self._session, vbd_ref) - - task1 = self._session.call_xenapi("Async.VM.hard_shutdown", - rescue_vm_ref) - self._session.wait_for_task(task1, instance.id) - - vdi_refs = VMHelper.lookup_vm_vdis(self._session, rescue_vm_ref) - for vdi_ref in vdi_refs: - try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) - self._session.wait_for_task(task, instance.id) - except self.XenAPI.Failure: - continue - - task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm_ref) - self._session.wait_for_task(task2, instance.id) - + self._destroy_rescue_vbds(rescue_vm_ref) + self._shutdown_rescue(rescue_vm_ref) + self._destroy_rescue_vdis(rescue_vm_ref) + self._destroy_rescue(rescue_vm_ref) self._release_bootlock(original_vm_ref) self._start(instance, original_vm_ref) + def poll_rescued_instances(self, timeout): + """Look for expirable rescued instances + - forcibly exit rescue mode for any instances that have been + in rescue mode for >= the provided timeout + """ + last_ran = self.poll_rescue_last_ran + if last_ran: + if not utils.is_then_greater(last_ran, timeout * 60 * 60): + # Do not run. Let's bail. + return + else: + # Update the time tracker and proceed. + self.poll_rescue_last_ran = utils.utcnow() + else: + # We need a base time to start tracking. + self.poll_rescue_last_ran = utils.utcnow() + return + + vms = [] + for instance in self.list_instances(): + if instance.endswith("-rescue"): + vms.append(dict(name=instance, + vm_ref=VMHelper.lookup(self._session, + instance))) + + for vm in vms: + rescue_name = vm["name"] + rescue_vm_ref = vm["vm_ref"] + original_name = vm["name"].split("-rescue", 1)[0] + original_vm_ref = VMHelper.lookup(self._session, original_name) + + self._destroy_rescue_vbds(rescue_vm_ref) + self._shutdown_rescue(rescue_vm_ref) + self._destroy_rescue_vdis(rescue_vm_ref) + self._destroy_rescue(rescue_vm_ref) + self._release_bootlock(original_vm_ref) + self._session.call_xenapi("VM.start", original_vm_ref, False, + False) + def get_info(self, instance): """Return data about VM instance""" vm_ref = self._get_vm_opaque_ref(instance) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index da42a83b6b..50aad96b88 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -225,6 +225,10 @@ class XenAPIConnection(object): """Unrescue the specified instance""" self._vmops.unrescue(instance, callback) + def poll_rescued_instances(self, timeout): + """Poll for rescued instances""" + self._vmops.poll_rescued_instances(timeout) + def reset_network(self, instance): """reset networking for specified instance""" self._vmops.reset_network(instance) From 8eab4f6ecaf51221b335e76d9e532a1f159c2f2d Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 23 Mar 2011 19:44:32 +0000 Subject: [PATCH 091/101] Forgot extraneous module import --- nova/api/openstack/servers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f3367e118a..d392ab57fb 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -23,7 +23,6 @@ from webob import exc from nova import compute from nova import context -from nova import db from nova import exception from nova import flags from nova import log as logging From 0218a11bb1d5275d5b99c98aea1edba0f45f56e2 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 23 Mar 2011 19:48:26 +0000 Subject: [PATCH 092/101] Forgot extraneous module import again --- nova/api/openstack/views/servers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 18d31a29d9..68f712e56d 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -20,7 +20,6 @@ import hashlib from nova.compute import power_state import nova.compute import nova.context -from nova import db from nova.api.openstack import common from nova.api.openstack.views import addresses as addresses_view from nova.api.openstack.views import flavors as flavors_view From a291e68fef876080d7984a1d7192e939808596bf Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 14:55:33 -0500 Subject: [PATCH 093/101] Fixed some typos --- nova/virt/xenapi/vmops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index cb36730a0b..0a516bd36a 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -463,7 +463,7 @@ class VMOps(object): except self.XenAPI.Failure, exc: LOG.exception(exc) - def _shutdown_rescue(self, vm_ref): + def _shutdown_rescue(self, rescue_vm_ref): """Shutdown a rescue instance""" self._session.call_xenapi("Async.VM.hard_shutdown", rescue_vm_ref) @@ -552,7 +552,7 @@ class VMOps(object): LOG.debug(_("Instance %(instance_id)s VM destroyed") % locals()) - def _destroy_rescue(self, vm_ref): + def _destroy_rescue_instance(self, rescue_vm_ref): """Destroy a rescue instance""" self._session.call_xenapi("Async.VM.destroy", rescue_vm_ref) @@ -671,7 +671,7 @@ class VMOps(object): self._destroy_rescue_vbds(rescue_vm_ref) self._shutdown_rescue(rescue_vm_ref) self._destroy_rescue_vdis(rescue_vm_ref) - self._destroy_rescue(rescue_vm_ref) + self._destroy_rescue_instance(rescue_vm_ref) self._release_bootlock(original_vm_ref) self._start(instance, original_vm_ref) @@ -709,7 +709,7 @@ class VMOps(object): self._destroy_rescue_vbds(rescue_vm_ref) self._shutdown_rescue(rescue_vm_ref) self._destroy_rescue_vdis(rescue_vm_ref) - self._destroy_rescue(rescue_vm_ref) + self._destroy_rescue_instance(rescue_vm_ref) self._release_bootlock(original_vm_ref) self._session.call_xenapi("VM.start", original_vm_ref, False, False) From 83e519b734078d8214fa0dc1d518607c7c0b244a Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 15:21:18 -0500 Subject: [PATCH 094/101] Only run periodic task when rescue_timeout is greater than 0 --- nova/compute/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3834c33ab1..9cb210c772 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -138,7 +138,8 @@ class ComputeManager(manager.Manager): def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval.""" super(ComputeManager, self).periodic_tasks(context) - self.driver.poll_rescued_instances(FLAGS.rescue_timeout) + if FLAGS.rescue_timeout > 0: + self.driver.poll_rescued_instances(FLAGS.rescue_timeout) def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" From 0d677a9b63ed9b4612379494bf8a58af1c090331 Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Wed, 23 Mar 2011 16:51:30 -0400 Subject: [PATCH 095/101] Should not call super __init__ twice in APIRouter --- nova/api/openstack/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index e68110bc4b..143b1d2b2e 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -127,8 +127,6 @@ class APIRouter(wsgi.Router): _limits = limits.LimitsController() mapper.resource("limit", "limits", controller=_limits) - super(APIRouter, self).__init__(mapper) - class APIRouterV10(APIRouter): """Define routes specific to OpenStack API V1.0.""" From 98b4f0924257dcfa12e4881950472e983f08ef1d Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 23 Mar 2011 21:04:42 +0000 Subject: [PATCH 096/101] merge prop fixes --- nova/compute/api.py | 10 +++++++--- nova/tests/test_compute.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index c2738f6f5f..01eead4ac9 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -260,7 +260,7 @@ class API(base.Base): db.migration_get_by_instance_and_status(context, instance_id, 'finished') return True - except Exception, e: + except exception.NotFound: return False def ensure_default_security_group(self, context): @@ -512,10 +512,14 @@ class API(base.Base): raise exception.ApiError(_("Requested flavor %(flavor_id)d " "does not exist") % locals()) - if current_instance_type['memory_mb'] >= \ - new_instance_type['memory_mb']: + current_memory_mb = current_instance_type['memory_mb'] + new_memory_mb = new_instance_type['memory_mb'] + if current_memory_mb > new_memory_mb: raise exception.ApiError(_("Invalid flavor: cannot downsize" "instances")) + if current_memory_mb == new_memory_mb: + raise exception.ApiError(_("Invalid flavor: cannot use" + "the same flavor. ")) self._cast_scheduler_message(context, {"method": "prep_resize", diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 444be5dd8b..44d04a12fe 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -336,7 +336,7 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(context, instance_id) def test_resize_down_fails(self): - """Ensure invalid flavors raise""" + """Ensure resizing down raises and fails""" context = self.context.elevated() instance_id = self._create_instance() @@ -349,6 +349,18 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(context, instance_id) + def test_resize_same_size_fails(self): + """Ensure invalid flavors raise""" + context = self.context.elevated() + instance_id = self._create_instance() + + self.compute.run_instance(self.context, instance_id) + + self.assertRaises(exception.ApiError, self.compute_api.resize, + context, instance_id, 1) + + self.compute.terminate_instance(context, instance_id) + def test_get_by_flavor_id(self): type = instance_types.get_by_flavor_id(1) self.assertEqual(type, 'm1.tiny') From b3a8c70304672abe9b461c6cfeed3e8b517ca0b6 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 16:56:54 -0500 Subject: [PATCH 097/101] Added docstring --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/utils.py b/nova/utils.py index 38cdb80219..bf1aa4a916 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -335,6 +335,7 @@ utcnow.override_time = None def is_then_greater(then, seconds): + """Return True of 'then' is greater than 'seconds'""" if utcnow() - then > datetime.timedelta(seconds=seconds): return True else: From a12b6f0a0808fba5541723a537118447b55b69ad Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 17:15:41 -0500 Subject: [PATCH 098/101] Better method name --- nova/utils.py | 6 +++--- nova/virt/xenapi/vmops.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index bf1aa4a916..04b6c9778e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -334,9 +334,9 @@ def utcnow(): utcnow.override_time = None -def is_then_greater(then, seconds): - """Return True of 'then' is greater than 'seconds'""" - if utcnow() - then > datetime.timedelta(seconds=seconds): +def is_older_than(before, seconds): + """Return True if before is older than 'seconds'""" + if utcnow() - before > datetime.timedelta(seconds=seconds): return True else: return False diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 0a516bd36a..3f1eceddce 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -682,7 +682,7 @@ class VMOps(object): """ last_ran = self.poll_rescue_last_ran if last_ran: - if not utils.is_then_greater(last_ran, timeout * 60 * 60): + if not utils.is_older_than(last_ran, timeout * 60 * 60): # Do not run. Let's bail. return else: From e19b12f668fb6cd693df6834f8895fb5487953d7 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 18:34:47 -0500 Subject: [PATCH 099/101] Review feedback --- nova/compute/manager.py | 2 +- nova/virt/xenapi/vmops.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 9cb210c772..ce1ae87e32 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -68,7 +68,7 @@ flags.DEFINE_integer('live_migration_retry_count', 30, "Retry count needed in live_migration." " sleep 1 sec for each count") flags.DEFINE_integer("rescue_timeout", 0, - "Automatically unrescue an instance after N hours." + "Automatically unrescue an instance after N seconds." " Set to 0 to disable.") LOG = logging.getLogger('nova.compute.manager') diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3f1eceddce..1f2e10aa60 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -497,8 +497,8 @@ class VMOps(object): """Destroys all VBDs tied to a rescue VM""" vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) for vbd_ref in vbd_refs: - _vbd_ref = self._session.get_xenapi().VBD.get_record(vbd_ref) - if _vbd_ref["userdevice"] == "1": + vbd_rec = self._session.get_xenapi().VBD.get_record(vbd_ref) + if vbd_rec["userdevice"] == "1": # primary VBD is always 1 VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) @@ -554,6 +554,10 @@ class VMOps(object): def _destroy_rescue_instance(self, rescue_vm_ref): """Destroy a rescue instance""" + self._destroy_rescue_vbds(rescue_vm_ref) + self._shutdown_rescue(rescue_vm_ref) + self._destroy_rescue_vdis(rescue_vm_ref) + self._session.call_xenapi("Async.VM.destroy", rescue_vm_ref) def destroy(self, instance): @@ -668,9 +672,6 @@ class VMOps(object): original_vm_ref = self._get_vm_opaque_ref(instance) instance._rescue = False - self._destroy_rescue_vbds(rescue_vm_ref) - self._shutdown_rescue(rescue_vm_ref) - self._destroy_rescue_vdis(rescue_vm_ref) self._destroy_rescue_instance(rescue_vm_ref) self._release_bootlock(original_vm_ref) self._start(instance, original_vm_ref) @@ -682,7 +683,7 @@ class VMOps(object): """ last_ran = self.poll_rescue_last_ran if last_ran: - if not utils.is_older_than(last_ran, timeout * 60 * 60): + if not utils.is_older_than(last_ran, timeout): # Do not run. Let's bail. return else: @@ -693,23 +694,22 @@ class VMOps(object): self.poll_rescue_last_ran = utils.utcnow() return - vms = [] + rescue_vms = [] for instance in self.list_instances(): if instance.endswith("-rescue"): - vms.append(dict(name=instance, - vm_ref=VMHelper.lookup(self._session, - instance))) + rescue_vms.append(dict(name=instance, + vm_ref=VMHelper.lookup(self._session, + instance))) - for vm in vms: + for vm in rescue_vms: rescue_name = vm["name"] rescue_vm_ref = vm["vm_ref"] + + self._destroy_rescue_instance(rescue_vm_ref) + original_name = vm["name"].split("-rescue", 1)[0] original_vm_ref = VMHelper.lookup(self._session, original_name) - self._destroy_rescue_vbds(rescue_vm_ref) - self._shutdown_rescue(rescue_vm_ref) - self._destroy_rescue_vdis(rescue_vm_ref) - self._destroy_rescue_instance(rescue_vm_ref) self._release_bootlock(original_vm_ref) self._session.call_xenapi("VM.start", original_vm_ref, False, False) From 10e61af8a23c126c15fcfcf25156d32facf19ec2 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 23 Mar 2011 22:55:04 -0500 Subject: [PATCH 100/101] Added hyperv stub --- nova/virt/hyperv.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/virt/hyperv.py b/nova/virt/hyperv.py index 29d18dac5f..75fed6d4f6 100644 --- a/nova/virt/hyperv.py +++ b/nova/virt/hyperv.py @@ -467,3 +467,6 @@ class HyperVConnection(object): if vm is None: raise exception.NotFound('Cannot detach volume from missing %s ' % instance_name) + + def poll_rescued_instances(self, timeout): + pass From f52a2a8a440b303e5289815ab4f6c2d24bfdc59f Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Thu, 24 Mar 2011 01:41:38 -0400 Subject: [PATCH 101/101] Fixed the docstring for common.get_id_from_href --- nova/api/openstack/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 21ceec45e2..bff050347b 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -79,10 +79,10 @@ def get_image_id_from_image_hash(image_service, context, image_hash): def get_id_from_href(href): - """Return the id portion of a url. + """Return the id portion of a url as an int. Given: http://www.foo.com/bar/123?q=4 - Returns: 4 + Returns: 123 """ try: