db: Add initial alembic migration for main DB

This wasn't as complicated as feared. We're mostly able to copy-paste
the existing sqlalchemy-migrate migration and simply changes the pattern
of calls from monkey-patched 'create' methods such as:

  agent_builds = Table('agent_builds', meta,
      sa.Column('created_at', sa.DateTime),
      ...
      mysql_charset='utf8'
  )
  agent_builds.create()

to explicit alembic APIs like:

  op.create_table(
      'agent_builds',
      sa.Column('created_at', sa.DateTime),
      ...
      mysql_charset='utf8'
  )

Reviewers are encouraged to diff the old migration file,
'nova/db/main/legacy_migrations/versions/402_train.py' against the new
one, 'nova/db/main/migrations/versions/8f2f1571d55b_initial_version.py',
to ease their job. The only significant divergences are the removal of
a single reference to the 'migrate_version' table created by
sqlalchemy-migrate, which obviously won't exist in an alembic-only
world, along with some reordering of tables. The latter step is
necessary since Alembic is not smart enough to correctly order the
creation of tables so that tables that reference (via a foreign key) one
or more other tables are created after the table(s) they reference.
Since we now have to create tables using the 'create_table' API as noted
above, rather than by creating a 'Table' instance and calling the
sqlalchemy-migrate-provided 'create' API as we could previously, we must
reorder our calls. The alternative would be to leave the creation of any
'ForeignKeyConstraint' until later but that seems no better and would
arguably be harder to read.

Note that this isn't yet wired up to anything: users can run it manually
but the nova-manage commands we provide are still connected to the
sqlalchemy-migrate commands. This will change shortly when we add shims
to handle the conversion.

Change-Id: I1fa2feaee78213ad81f1889ce54888696f58d98c
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>

nova/db/main/alembic.ini

@@ -0,0 +1,89 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = nova/db/main/migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to nova/db/main/migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat nova/db/main/migrations/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = sqlite:///nova.db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

nova/db/main/legacy_migrations/versions/402_train.py

@@ -25,9 +25,6 @@ from nova.objects import keypair
 LOG = logging.getLogger(__name__)
 
 
-# Note on the autoincrement flag: this is defaulted for primary key columns
-# of integral type, so is no longer set explicitly in such cases.
-
 # NOTE(dprince): This wrapper allows us to easily match the Folsom MySQL
 # Schema. In Folsom we created tables as latin1 and converted them to utf8
 # later. This conversion causes some of the Text columns on MySQL to get

nova/db/main/migrations/README.rst

@@ -0,0 +1,15 @@
+Migrations for the main database
+================================
+
+This directory contains migrations for the main database. These are implemented
+using `alembic`__, a lightweight database migration tool designed for usage
+with `SQLAlchemy`__.
+
+The best place to start understanding Alembic is with its own `tutorial`__. You
+can also play around with the :command:`alembic` command::
+
+    $ alembic --help
+
+.. __: https://alembic.sqlalchemy.org/en/latest/
+.. __: https://www.sqlalchemy.org/
+.. __: https://alembic.sqlalchemy.org/en/latest/tutorial.html

nova/db/main/migrations/env.py

@@ -0,0 +1,79 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL and not an Engine, though an
+    Engine is acceptable here as well.  By skipping the Engine creation we
+    don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the script output.
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine and associate a connection
+    with the context.
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

nova/db/main/migrations/script.py.mako

@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}