add support for database migration

bin/nova-manage

@@ -82,6 +82,7 @@ from nova import quota
 from nova import utils
 from nova.auth import manager
 from nova.cloudpipe import pipelib
+from nova.db import migration
 
 
 logging.basicConfig()
@@ -515,6 +516,22 @@ class LogCommands(object):
         print re.sub('#012', "\n", "\n".join(lines))
 
 
+class DbCommands(object):
+    """Class for managing the database."""
+
+    def __init__(self):
+        pass
+
+    def sync(self, version=None):
+        """adds role to user
+        if project is specified, adds project specific role
+        arguments: user, role [project]"""
+        return migration.db_sync(version)
+
+    def version(self):
+        print migration.db_version()
+
+
 CATEGORIES = [
     ('user', UserCommands),
     ('project', ProjectCommands),
@@ -524,7 +541,8 @@ CATEGORIES = [
     ('floating', FloatingIpCommands),
     ('network', NetworkCommands),
     ('service', ServiceCommands),
-    ('log', LogCommands)]
+    ('log', LogCommands),
+    ('db', DbCommands)]
 
 
 def lazy_match(name, key_value_tuples):

nova/db/migration.py

@@ -0,0 +1,38 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Database setup and migration commands."""
+
+from nova import flags
+from nova import utils
+
+FLAGS = flags.FLAGS
+flags.DECLARE('db_backend', 'nova.db.api')
+
+
+IMPL = utils.LazyPluggable(FLAGS['db_backend'],
+                           sqlalchemy='nova.db.sqlalchemy.migration')
+
+
+def db_sync(version=None):
+    """Migrate the database to `version` or the most recent version."""
+    return IMPL.db_sync(version=version)
+
+
+def db_version():
+    """Display the current database version."""
+    return IMPL.db_version()

nova/db/sqlalchemy/__init__.py

@@ -15,31 +15,3 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
-"""
-SQLAlchemy database backend
-"""
-import time
-
-from sqlalchemy.exc import OperationalError
-
-from nova import flags
-from nova import log as logging
-from nova.db.sqlalchemy import models
-
-
-FLAGS = flags.FLAGS
-LOG = logging.getLogger('nova.db.sqlalchemy')
-
-
-for i in xrange(FLAGS.sql_max_retries):
-    if i > 0:
-        time.sleep(FLAGS.sql_retry_interval)
-
-    try:
-        models.register_models()
-        break
-    except OperationalError:
-        LOG.exception(_("Data store %s is unreachable."
-                        " Trying again in %d seconds."),
-                      FLAGS.sql_connection, FLAGS.sql_retry_interval)

nova/db/sqlalchemy/migrate_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

nova/db/sqlalchemy/migrate_repo/manage.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+if __name__ == '__main__':
+  main(debug='False', repository='.')

nova/db/sqlalchemy/migrate_repo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=nova
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

nova/db/sqlalchemy/migrate_repo/versions/001_first_database.py

@@ -0,0 +1,547 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+## Table code mostly autogenerated by genmodel.py
+from sqlalchemy import *
+from migrate import *
+
+from nova import log as logging
+
+
+meta = MetaData()
+
+
+auth_tokens = Table('auth_tokens', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('token_hash',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        Column('user_id', Integer()),
+        Column('server_manageent_url',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('storage_url',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('cdn_management_url',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+export_devices = Table('export_devices', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('shelf_id', Integer()),
+        Column('blade_id', Integer()),
+        Column('volume_id',
+               Integer(),
+               ForeignKey('volumes.id'),
+               nullable=True),
+        )
+
+
+fixed_ips = Table('fixed_ips', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('address',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('network_id',
+               Integer(),
+               ForeignKey('networks.id'),
+               nullable=True),
+        Column('instance_id',
+               Integer(),
+               ForeignKey('instances.id'),
+               nullable=True),
+        Column('allocated', Boolean(create_constraint=True, name=None)),
+        Column('leased', Boolean(create_constraint=True, name=None)),
+        Column('reserved', Boolean(create_constraint=True, name=None)),
+        )
+
+
+floating_ips = Table('floating_ips', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('address',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('fixed_ip_id',
+               Integer(),
+               ForeignKey('fixed_ips.id'),
+               nullable=True),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('host',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+instances = Table('instances', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('internal_id', Integer()),
+        Column('admin_pass',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('image_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('kernel_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('ramdisk_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('launch_index', Integer()),
+        Column('key_name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('key_data',
+               Text(length=None, convert_unicode=False, assert_unicode=None,
+                    unicode_error=None, _warn_on_bytestring=False)),
+        Column('state', Integer()),
+        Column('state_description',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('memory_mb', Integer()),
+        Column('vcpus', Integer()),
+        Column('local_gb', Integer()),
+        Column('hostname',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('host',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('instance_type',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('user_data',
+               Text(length=None, convert_unicode=False, assert_unicode=None,
+                    unicode_error=None, _warn_on_bytestring=False)),
+        Column('reservation_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('mac_address',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('scheduled_at', DateTime(timezone=False)),
+        Column('launched_at', DateTime(timezone=False)),
+        Column('terminated_at', DateTime(timezone=False)),
+        Column('display_name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('display_description',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+iscsi_targets = Table('iscsi_targets', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('target_num', Integer()),
+        Column('host',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('volume_id',
+               Integer(),
+               ForeignKey('volumes.id'),
+               nullable=True),
+        )
+
+
+key_pairs = Table('key_pairs', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('fingerprint',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('public_key',
+               Text(length=None, convert_unicode=False, assert_unicode=None,
+                    unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+networks = Table('networks', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('injected', Boolean(create_constraint=True, name=None)),
+        Column('cidr',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('netmask',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('bridge',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('gateway',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('broadcast',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('dns',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('vlan', Integer()),
+        Column('vpn_public_address',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('vpn_public_port', Integer()),
+        Column('vpn_private_address',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('dhcp_start',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('host',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+projects = Table('projects', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        Column('name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('description',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('project_manager',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               ForeignKey('users.id')),
+        )
+
+
+quotas = Table('quotas', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('instances', Integer()),
+        Column('cores', Integer()),
+        Column('volumes', Integer()),
+        Column('gigabytes', Integer()),
+        Column('floating_ips', Integer()),
+        )
+
+
+security_groups = Table('security_groups', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('description',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+security_group_inst_assoc = Table('security_group_instance_association', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(), primary_key=True, nullable=False),
+        Column('security_group_id',
+               Integer(),
+               ForeignKey('security_groups.id')),
+        Column('instance_id', Integer(), ForeignKey('instances.id')),
+        )
+
+
+security_group_rules = Table('security_group_rules', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('parent_group_id',
+               Integer(),
+               ForeignKey('security_groups.id')),
+        Column('protocol',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('from_port', Integer()),
+        Column('to_port', Integer()),
+        Column('cidr',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('group_id',
+               Integer(),
+               ForeignKey('security_groups.id')),
+        )
+
+
+services = Table('services', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('host',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('binary',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('topic',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('report_count', Integer(),  nullable=False),
+        Column('disabled', Boolean(create_constraint=True, name=None)),
+        )
+
+
+users = Table('users', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        Column('name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('access_key',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('secret_key',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('is_admin', Boolean(create_constraint=True, name=None)),
+        )
+
+
+user_project_association = Table('user_project_association', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               ForeignKey('users.id'),
+               primary_key=True,
+               nullable=False),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               ForeignKey('projects.id'),
+               primary_key=True,
+               nullable=False),
+        )
+
+
+user_project_role_association = Table('user_project_role_association', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        Column('role',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        ForeignKeyConstraint(['user_id',
+                              'project_id'],
+                             ['user_project_association.user_id',
+                              'user_project_association.project_id']),
+        )
+
+
+user_role_association = Table('user_role_association', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               ForeignKey('users.id'),
+               primary_key=True,
+               nullable=False),
+        Column('role',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        )
+
+
+volumes = Table('volumes', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('ec2_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('host',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('size', Integer()),
+        Column('availability_zone',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('instance_id',
+               Integer(),
+               ForeignKey('instances.id'),
+               nullable=True),
+        Column('mountpoint',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('attach_time',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('status',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('attach_status',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('scheduled_at', DateTime(timezone=False)),
+        Column('launched_at', DateTime(timezone=False)),
+        Column('terminated_at', DateTime(timezone=False)),
+        Column('display_name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('display_description',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine;
+    # bind migrate_engine to your metadata
+    meta.bind = migrate_engine
+
+    for table in (auth_tokens, export_devices, fixed_ips, floating_ips,
+                  instances, iscsi_targets, key_pairs, networks,
+                  projects, quotas, security_groups, security_group_inst_assoc,
+                  security_group_rules, services, users,
+                  user_project_association, user_project_role_association,
+                  user_role_association, volumes):
+        try:
+            table.create()
+        except Exception:
+            logging.info(repr(table))
+            logging.exception('Exception while creating table')
+            raise
+
+
+def downgrade(migrate_engine):
+    # Operations to reverse the above upgrade go here.
+    for table in (auth_tokens, export_devices, fixed_ips, floating_ips,
+                  instances, iscsi_targets, key_pairs, networks,
+                  projects, quotas, security_groups, security_group_inst_assoc,
+                  security_group_rules, services, users,
+                  user_project_association, user_project_role_association,
+                  user_role_association, volumes):
+        table.drop()

nova/db/sqlalchemy/migrate_repo/versions/002_update_to_trunk.py

@@ -0,0 +1,125 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import *
+from migrate import *
+
+from nova import log as logging
+
+
+meta = MetaData()
+
+
+# Just for the ForeignKey to succeed
+instances = Table('instances', meta,
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        )
+
+#
+# New Tables
+#
+instance_actions = Table('instance_actions', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('instance_id',
+               Integer(),
+               ForeignKey('instances.id')),
+        Column('action',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('error',
+               Text(length=None, convert_unicode=False, assert_unicode=None,
+                    unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+certificates = Table('certificates', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        Column('user_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('project_id',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('file_name',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+#
+# Tables to alter
+#
+auth_tokens = Table('auth_tokens', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('token_hash',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False),
+               primary_key=True,
+               nullable=False),
+        Column('user_id', Integer()),
+        Column('server_manageent_url',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('storage_url',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('cdn_management_url',
+               String(length=None, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        )
+
+
+instances_availability_zone = Column(
+        'availability_zone',
+        String(length=None, convert_unicode=False, assert_unicode=None,
+               unicode_error=None, _warn_on_bytestring=False))
+
+
+instances_locked = Column('locked',
+                          Boolean(create_constraint=True, name=None))
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine;
+    # bind migrate_engine to your metadata
+    meta.bind = migrate_engine
+    for table in (instance_actions, certificates):
+        try:
+            table.create()
+        except Exception:
+            logging.info(repr(table))
+            logging.exception('Exception while creating table')
+            raise
+
+    auth_tokens.c.user_id.alter(type=String(length=None,
+                                            convert_unicode=False,
+                                            assert_unicode=None,
+                                            unicode_error=None,
+                                            _warn_on_bytestring=False))
+    instances_availability_zone.create(table=instances)
+    instances_locked.create(table=instances)

nova/db/sqlalchemy/migration.py

@@ -0,0 +1,72 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from nova import flags
+
+import sqlalchemy
+from migrate.versioning import api as versioning_api
+from migrate.versioning import exceptions as versioning_exceptions
+
+FLAGS = flags.FLAGS
+
+
+def db_sync(version=None):
+    db_version()
+    repo_path = _find_migrate_repo()
+    return versioning_api.upgrade(FLAGS.sql_connection, repo_path, version)
+
+
+def db_version():
+    repo_path = _find_migrate_repo()
+    try:
+        return versioning_api.db_version(FLAGS.sql_connection, repo_path)
+    except versioning_exceptions.DatabaseNotControlledError:
+        # If we aren't version controlled we may already have the database
+        # in the state from before we started version control, check for that
+        # and set up version_control appropriately
+        meta = sqlalchemy.MetaData()
+        engine = sqlalchemy.create_engine(FLAGS.sql_connection, echo=False)
+        meta.reflect(bind=engine)
+        try:
+            for table in ('auth_tokens', 'export_devices', 'fixed_ips',
+                          'floating_ips', 'instances', 'iscsi_targets',
+                          'key_pairs', 'networks', 'projects', 'quotas',
+                          'security_group_rules',
+                          'security_group_instance_association', 'services',
+                          'users', 'user_project_association',
+                          'user_project_role_association', 'volumes'):
+                assert table in meta.tables
+            return db_version_control(1)
+        except AssertionError:
+            return db_version_control(0)
+
+
+def db_version_control(version=None):
+    repo_path = _find_migrate_repo()
+    versioning_api.version_control(FLAGS.sql_connection, repo_path, version)
+    return version
+
+
+def _find_migrate_repo():
+    """Get the path for the migrate repository."""
+    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                        'migrate_repo')
+    assert os.path.exists(path)
+    return path

nova/db/sqlalchemy/models.py

@@ -94,51 +94,6 @@ class NovaBase(object):
         return iter(self)
 
 
-# TODO(vish): Store images in the database instead of file system
-#class Image(BASE, NovaBase):
-#    """Represents an image in the datastore"""
-#    __tablename__ = 'images'
-#    id = Column(Integer, primary_key=True)
-#    ec2_id = Column(String(12), unique=True)
-#    user_id = Column(String(255))
-#    project_id = Column(String(255))
-#    image_type = Column(String(255))
-#    public = Column(Boolean, default=False)
-#    state = Column(String(255))
-#    location = Column(String(255))
-#    arch = Column(String(255))
-#    default_kernel_id = Column(String(255))
-#    default_ramdisk_id = Column(String(255))
-#
-#    @validates('image_type')
-#    def validate_image_type(self, key, image_type):
-#        assert(image_type in ['machine', 'kernel', 'ramdisk', 'raw'])
-#
-#    @validates('state')
-#    def validate_state(self, key, state):
-#        assert(state in ['available', 'pending', 'disabled'])
-#
-#    @validates('default_kernel_id')
-#    def validate_kernel_id(self, key, val):
-#        if val != 'machine':
-#            assert(val is None)
-#
-#    @validates('default_ramdisk_id')
-#    def validate_ramdisk_id(self, key, val):
-#        if val != 'machine':
-#            assert(val is None)
-#
-#
-# TODO(vish): To make this into its own table, we need a good place to
-#             create the host entries. In config somwhere? Or the first
-#             time any object sets host? This only becomes particularly
-#             important if we need to store per-host data.
-#class Host(BASE, NovaBase):
-#    """Represents a host where services are running"""
-#    __tablename__ = 'hosts'
-#    id = Column(String(255), primary_key=True)
-
-
 class Service(BASE, NovaBase):
     """Represents a running service on a host."""
 

nova/service.py

@@ -209,19 +209,6 @@ class Service(object):
                 self.model_disconnected = True
                 logging.exception(_("model server went away"))
 
-                try:
-                    # NOTE(vish): This is late-loaded to make sure that the
-                    #             database is not created before flags have
-                    #             been loaded.
-                    from nova.db.sqlalchemy import models
-                    models.register_models()
-                except OperationalError:
-                    logging.exception(_("Data store %s is unreachable."
-                                        " Trying again in %d seconds.") %
-                                      (FLAGS.sql_connection,
-                                       FLAGS.sql_retry_interval))
-                    time.sleep(FLAGS.sql_retry_interval)
-
 
 def serve(*services):
     FLAGS(sys.argv)

run_tests.py

@@ -17,7 +17,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-
+import gettext
 import os
 import unittest
 import sys
@@ -26,6 +26,10 @@ from nose import config
 from nose import result
 from nose import core
 
+gettext.install('nova', unicode=1)
+
+from nova.db import migration
+
 
 class NovaTestResult(result.TextTestResult):
     def __init__(self, *args, **kw):
@@ -61,6 +65,9 @@ if __name__ == '__main__':
     c = config.Config(stream=sys.stdout,
                       env=os.environ,
                       verbosity=3)
+    
+    migration.db_sync()
+
 
     runner = NovaTestRunner(stream=c.stream,
                             verbosity=c.verbosity,

tools/pip-requires

@@ -25,3 +25,4 @@ bzr
 Twisted>=10.1.0
 PasteDeploy
 paste
+sqlalchemy-migrate