Revert "Switch Nova to use oslo.concurrency"

This patch made unit tests stop running (always runs 0 unit tests) we
should revert and put a check in place to prevent that from happening
so that we don't regress the rest of Nova trying to figure out the
oslo.concurrency issue.

This reverts commit 382294762b.

Change-Id: I98b04ac2a0ba67ba8af44867799317fdf95b8aff

nova/cmd/conductor.py

@@ -16,12 +16,12 @@
 
 import sys
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import config
 from nova import objects
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova.openstack.common.report import guru_meditation_report as gmr
 from nova import service
 from nova import utils

nova/console/xvp.py

@@ -19,7 +19,6 @@ import os
 import signal
 
 import jinja2
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import excutils
 
@@ -27,6 +26,7 @@ from nova import context
 from nova import db
 from nova.i18n import _, _LE
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import paths
 from nova import utils
 

nova/crypto.py

@@ -28,7 +28,6 @@ import re
 import string
 import struct
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import excutils
 from oslo.utils import timeutils
@@ -41,6 +40,7 @@ from nova import exception
 from nova.i18n import _
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import paths
 from nova import utils
 

nova/image/s3.py

@@ -26,7 +26,6 @@ import tempfile
 import boto.s3.connection
 import eventlet
 from lxml import etree
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova.api.ec2 import ec2utils
@@ -36,6 +35,7 @@ from nova import exception
 from nova.i18n import _, _LE
 from nova.image import glance
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 
 

nova/network/base_api.py

@@ -16,7 +16,6 @@
 import functools
 import inspect
 
-from oslo.concurrency import lockutils
 from oslo.utils import excutils
 
 from nova.db import base
@@ -24,6 +23,7 @@ from nova import hooks
 from nova.i18n import _
 from nova.network import model as network_model
 from nova import objects
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 
 

nova/network/floating_ips.py

@@ -15,7 +15,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo import messaging
 from oslo.utils import excutils
@@ -29,6 +28,7 @@ from nova.i18n import _, _LE
 from nova.network import rpcapi as network_rpcapi
 from nova import objects
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova.openstack.common import uuidutils
 from nova import quota
 from nova import rpc

nova/network/linux_net.py

@@ -23,7 +23,6 @@ import os
 import re
 
 import netaddr
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 from oslo.utils import excutils
@@ -36,6 +35,7 @@ from nova.i18n import _, _LE
 from nova import objects
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import paths
 from nova import utils
 

nova/network/neutronv2/__init__.py

@@ -15,9 +15,9 @@
 
 from neutronclient.common import exceptions
 from neutronclient.v2_0 import client as clientv20
-from oslo.concurrency import lockutils
 from oslo.config import cfg
 
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 
 CONF = cfg.CONF

nova/network/neutronv2/api.py

@@ -19,7 +19,6 @@ import time
 import uuid
 
 from neutronclient.common import exceptions as neutron_client_exc
-from oslo.concurrency import lockutils
 from oslo.config import cfg
 from oslo.utils import excutils
 import six
@@ -34,6 +33,7 @@ from nova.network import model as network_model
 from nova.network import neutronv2
 from nova.network.neutronv2 import constants
 from nova import objects
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import uuidutils
 from nova.pci import manager as pci_manager

nova/openstack/common/lockutils.py

@@ -0,0 +1,326 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import contextlib
+import errno
+import functools
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import weakref
+
+from oslo.config import cfg
+
+from nova.openstack.common import fileutils
+from nova.openstack.common._i18n import _, _LE, _LI
+
+
+LOG = logging.getLogger(__name__)
+
+
+util_opts = [
+    cfg.BoolOpt('disable_process_locking', default=False,
+                help='Enables or disables inter-process locks.'),
+    cfg.StrOpt('lock_path',
+               default=os.environ.get("NOVA_LOCK_PATH"),
+               help='Directory to use for lock files.')
+]
+
+
+CONF = cfg.CONF
+CONF.register_opts(util_opts)
+
+
+def set_defaults(lock_path):
+    cfg.set_defaults(util_opts, lock_path=lock_path)
+
+
+class _FileLock(object):
+    """Lock implementation which allows multiple locks, working around
+    issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
+    not require any cleanup. Since the lock is always held on a file
+    descriptor rather than outside of the process, the lock gets dropped
+    automatically if the process crashes, even if __exit__ is not executed.
+
+    There are no guarantees regarding usage by multiple green threads in a
+    single process here. This lock works only between processes. Exclusive
+    access between local threads should be achieved using the semaphores
+    in the @synchronized decorator.
+
+    Note these locks are released when the descriptor is closed, so it's not
+    safe to close the file descriptor while another green thread holds the
+    lock. Just opening and closing the lock file can break synchronisation,
+    so lock files must be accessed only using this abstraction.
+    """
+
+    def __init__(self, name):
+        self.lockfile = None
+        self.fname = name
+
+    def acquire(self):
+        basedir = os.path.dirname(self.fname)
+
+        if not os.path.exists(basedir):
+            fileutils.ensure_tree(basedir)
+            LOG.info(_LI('Created lock path: %s'), basedir)
+
+        self.lockfile = open(self.fname, 'w')
+
+        while True:
+            try:
+                # Using non-blocking locks since green threads are not
+                # patched to deal with blocking locking calls.
+                # Also upon reading the MSDN docs for locking(), it seems
+                # to have a laughable 10 attempts "blocking" mechanism.
+                self.trylock()
+                LOG.debug('Got file lock "%s"', self.fname)
+                return True
+            except IOError as e:
+                if e.errno in (errno.EACCES, errno.EAGAIN):
+                    # external locks synchronise things like iptables
+                    # updates - give it some time to prevent busy spinning
+                    time.sleep(0.01)
+                else:
+                    raise threading.ThreadError(_("Unable to acquire lock on"
+                                                  " `%(filename)s` due to"
+                                                  " %(exception)s") %
+                                                {'filename': self.fname,
+                                                    'exception': e})
+
+    def __enter__(self):
+        self.acquire()
+        return self
+
+    def release(self):
+        try:
+            self.unlock()
+            self.lockfile.close()
+            LOG.debug('Released file lock "%s"', self.fname)
+        except IOError:
+            LOG.exception(_LE("Could not release the acquired lock `%s`"),
+                          self.fname)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.release()
+
+    def exists(self):
+        return os.path.exists(self.fname)
+
+    def trylock(self):
+        raise NotImplementedError()
+
+    def unlock(self):
+        raise NotImplementedError()
+
+
+class _WindowsLock(_FileLock):
+    def trylock(self):
+        msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
+
+    def unlock(self):
+        msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
+
+
+class _FcntlLock(_FileLock):
+    def trylock(self):
+        fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+    def unlock(self):
+        fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
+
+
+if os.name == 'nt':
+    import msvcrt
+    InterProcessLock = _WindowsLock
+else:
+    import fcntl
+    InterProcessLock = _FcntlLock
+
+_semaphores = weakref.WeakValueDictionary()
+_semaphores_lock = threading.Lock()
+
+
+def _get_lock_path(name, lock_file_prefix, lock_path=None):
+    # NOTE(mikal): the lock name cannot contain directory
+    # separators
+    name = name.replace(os.sep, '_')
+    if lock_file_prefix:
+        sep = '' if lock_file_prefix.endswith('-') else '-'
+        name = '%s%s%s' % (lock_file_prefix, sep, name)
+
+    local_lock_path = lock_path or CONF.lock_path
+
+    if not local_lock_path:
+        raise cfg.RequiredOptError('lock_path')
+
+    return os.path.join(local_lock_path, name)
+
+
+def external_lock(name, lock_file_prefix=None, lock_path=None):
+    LOG.debug('Attempting to grab external lock "%(lock)s"',
+              {'lock': name})
+
+    lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
+
+    return InterProcessLock(lock_file_path)
+
+
+def remove_external_lock_file(name, lock_file_prefix=None):
+    """Remove an external lock file when it's not used anymore
+    This will be helpful when we have a lot of lock files
+    """
+    with internal_lock(name):
+        lock_file_path = _get_lock_path(name, lock_file_prefix)
+        try:
+            os.remove(lock_file_path)
+        except OSError:
+            LOG.info(_LI('Failed to remove file %(file)s'),
+                     {'file': lock_file_path})
+
+
+def internal_lock(name):
+    with _semaphores_lock:
+        try:
+            sem = _semaphores[name]
+            LOG.debug('Using existing semaphore "%s"', name)
+        except KeyError:
+            sem = threading.Semaphore()
+            _semaphores[name] = sem
+            LOG.debug('Created new semaphore "%s"', name)
+
+    return sem
+
+
+@contextlib.contextmanager
+def lock(name, lock_file_prefix=None, external=False, lock_path=None):
+    """Context based lock
+
+    This function yields a `threading.Semaphore` instance (if we don't use
+    eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
+    True, in which case, it'll yield an InterProcessLock instance.
+
+    :param lock_file_prefix: The lock_file_prefix argument is used to provide
+      lock files on disk with a meaningful prefix.
+
+    :param external: The external keyword argument denotes whether this lock
+      should work across multiple processes. This means that if two different
+      workers both run a method decorated with @synchronized('mylock',
+      external=True), only one of them will execute at a time.
+    """
+    int_lock = internal_lock(name)
+    with int_lock:
+        LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name})
+        try:
+            if external and not CONF.disable_process_locking:
+                ext_lock = external_lock(name, lock_file_prefix, lock_path)
+                with ext_lock:
+                    yield ext_lock
+            else:
+                yield int_lock
+        finally:
+            LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name})
+
+
+def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
+    """Synchronization decorator.
+
+    Decorating a method like so::
+
+        @synchronized('mylock')
+        def foo(self, *args):
+           ...
+
+    ensures that only one thread will execute the foo method at a time.
+
+    Different methods can share the same lock::
+
+        @synchronized('mylock')
+        def foo(self, *args):
+           ...
+
+        @synchronized('mylock')
+        def bar(self, *args):
+           ...
+
+    This way only one of either foo or bar can be executing at a time.
+    """
+
+    def wrap(f):
+        @functools.wraps(f)
+        def inner(*args, **kwargs):
+            try:
+                with lock(name, lock_file_prefix, external, lock_path):
+                    LOG.debug('Got semaphore / lock "%(function)s"',
+                              {'function': f.__name__})
+                    return f(*args, **kwargs)
+            finally:
+                LOG.debug('Semaphore / lock released "%(function)s"',
+                          {'function': f.__name__})
+        return inner
+    return wrap
+
+
+def synchronized_with_prefix(lock_file_prefix):
+    """Partial object generator for the synchronization decorator.
+
+    Redefine @synchronized in each project like so::
+
+        (in oslo.utils.py)
+        from nova.openstack.common import lockutils
+
+        synchronized = lockutils.synchronized_with_prefix('nova-')
+
+
+        (in nova/foo.py)
+        from nova import utils
+
+        @utils.synchronized('mylock')
+        def bar(self, *args):
+           ...
+
+    The lock_file_prefix argument is used to provide lock files on disk with a
+    meaningful prefix.
+    """
+
+    return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
+
+
+def main(argv):
+    """Create a dir for locks and pass it to command from arguments
+
+    If you run this:
+    python -m openstack.common.lockutils python setup.py testr <etc>
+
+    a temporary directory will be created for all your locks and passed to all
+    your tests in an environment variable. The temporary dir will be deleted
+    afterwards and the return value will be preserved.
+    """
+
+    lock_dir = tempfile.mkdtemp()
+    os.environ["NOVA_LOCK_PATH"] = lock_dir
+    try:
+        ret_val = subprocess.call(argv[1:])
+    finally:
+        shutil.rmtree(lock_dir, ignore_errors=True)
+    return ret_val
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

nova/openstack/common/processutils.py

@@ -0,0 +1,289 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+System-level utilities and helper functions.
+"""
+
+import errno
+import logging
+import multiprocessing
+import os
+import random
+import shlex
+import signal
+
+from eventlet.green import subprocess
+from eventlet import greenthread
+from oslo.utils import strutils
+import six
+
+from nova.openstack.common._i18n import _
+
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidArgumentError(Exception):
+    def __init__(self, message=None):
+        super(InvalidArgumentError, self).__init__(message)
+
+
+class UnknownArgumentError(Exception):
+    def __init__(self, message=None):
+        super(UnknownArgumentError, self).__init__(message)
+
+
+class ProcessExecutionError(Exception):
+    def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
+                 description=None):
+        self.exit_code = exit_code
+        self.stderr = stderr
+        self.stdout = stdout
+        self.cmd = cmd
+        self.description = description
+
+        if description is None:
+            description = _("Unexpected error while running command.")
+        if exit_code is None:
+            exit_code = '-'
+        message = _('%(description)s\n'
+                    'Command: %(cmd)s\n'
+                    'Exit code: %(exit_code)s\n'
+                    'Stdout: %(stdout)r\n'
+                    'Stderr: %(stderr)r') % {'description': description,
+                                             'cmd': cmd,
+                                             'exit_code': exit_code,
+                                             'stdout': stdout,
+                                             'stderr': stderr}
+        super(ProcessExecutionError, self).__init__(message)
+
+
+class NoRootWrapSpecified(Exception):
+    def __init__(self, message=None):
+        super(NoRootWrapSpecified, self).__init__(message)
+
+
+def _subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+
+def execute(*cmd, **kwargs):
+    """Helper method to shell out and execute a command through subprocess.
+
+    Allows optional retry.
+
+    :param cmd:             Passed to subprocess.Popen.
+    :type cmd:              string
+    :param process_input:   Send to opened process.
+    :type process_input:    string
+    :param env_variables:   Environment variables and their values that
+                            will be set for the process.
+    :type env_variables:    dict
+    :param check_exit_code: Single bool, int, or list of allowed exit
+                            codes.  Defaults to [0].  Raise
+                            :class:`ProcessExecutionError` unless
+                            program exits with one of these code.
+    :type check_exit_code:  boolean, int, or [int]
+    :param delay_on_retry:  True | False. Defaults to True. If set to True,
+                            wait a short amount of time before retrying.
+    :type delay_on_retry:   boolean
+    :param attempts:        How many times to retry cmd.
+    :type attempts:         int
+    :param run_as_root:     True | False. Defaults to False. If set to True,
+                            the command is prefixed by the command specified
+                            in the root_helper kwarg.
+    :type run_as_root:      boolean
+    :param root_helper:     command to prefix to commands called with
+                            run_as_root=True
+    :type root_helper:      string
+    :param shell:           whether or not there should be a shell used to
+                            execute this command. Defaults to false.
+    :type shell:            boolean
+    :param loglevel:        log level for execute commands.
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
+    :returns:               (stdout, stderr) from process execution
+    :raises:                :class:`UnknownArgumentError` on
+                            receiving unknown arguments
+    :raises:                :class:`ProcessExecutionError`
+    """
+
+    process_input = kwargs.pop('process_input', None)
+    env_variables = kwargs.pop('env_variables', None)
+    check_exit_code = kwargs.pop('check_exit_code', [0])
+    ignore_exit_code = False
+    delay_on_retry = kwargs.pop('delay_on_retry', True)
+    attempts = kwargs.pop('attempts', 1)
+    run_as_root = kwargs.pop('run_as_root', False)
+    root_helper = kwargs.pop('root_helper', '')
+    shell = kwargs.pop('shell', False)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
+
+    if isinstance(check_exit_code, bool):
+        ignore_exit_code = not check_exit_code
+        check_exit_code = [0]
+    elif isinstance(check_exit_code, int):
+        check_exit_code = [check_exit_code]
+
+    if kwargs:
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
+
+    if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
+        if not root_helper:
+            raise NoRootWrapSpecified(
+                message=_('Command requested root, but did not '
+                          'specify a root helper.'))
+        cmd = shlex.split(root_helper) + list(cmd)
+
+    cmd = map(str, cmd)
+    sanitized_cmd = strutils.mask_password(' '.join(cmd))
+
+    while attempts > 0:
+        attempts -= 1
+        try:
+            LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd)
+            _PIPE = subprocess.PIPE  # pylint: disable=E1101
+
+            if os.name == 'nt':
+                preexec_fn = None
+                close_fds = False
+            else:
+                preexec_fn = _subprocess_setup
+                close_fds = True
+
+            obj = subprocess.Popen(cmd,
+                                   stdin=_PIPE,
+                                   stdout=_PIPE,
+                                   stderr=_PIPE,
+                                   close_fds=close_fds,
+                                   preexec_fn=preexec_fn,
+                                   shell=shell,
+                                   env=env_variables)
+            result = None
+            for _i in six.moves.range(20):
+                # NOTE(russellb) 20 is an arbitrary number of retries to
+                # prevent any chance of looping forever here.
+                try:
+                    if process_input is not None:
+                        result = obj.communicate(process_input)
+                    else:
+                        result = obj.communicate()
+                except OSError as e:
+                    if e.errno in (errno.EAGAIN, errno.EINTR):
+                        continue
+                    raise
+                break
+            obj.stdin.close()  # pylint: disable=E1101
+            _returncode = obj.returncode  # pylint: disable=E1101
+            LOG.log(loglevel, 'Result was %s' % _returncode)
+            if not ignore_exit_code and _returncode not in check_exit_code:
+                (stdout, stderr) = result
+                sanitized_stdout = strutils.mask_password(stdout)
+                sanitized_stderr = strutils.mask_password(stderr)
+                raise ProcessExecutionError(exit_code=_returncode,
+                                            stdout=sanitized_stdout,
+                                            stderr=sanitized_stderr,
+                                            cmd=sanitized_cmd)
+            return result
+        except ProcessExecutionError:
+            if not attempts:
+                raise
+            else:
+                LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd)
+                if delay_on_retry:
+                    greenthread.sleep(random.randint(20, 200) / 100.0)
+        finally:
+            # NOTE(termie): this appears to be necessary to let the subprocess
+            #               call clean something up in between calls, without
+            #               it two execute calls in a row hangs the second one
+            greenthread.sleep(0)
+
+
+def trycmd(*args, **kwargs):
+    """A wrapper around execute() to more easily handle warnings and errors.
+
+    Returns an (out, err) tuple of strings containing the output of
+    the command's stdout and stderr.  If 'err' is not empty then the
+    command can be considered to have failed.
+
+    :discard_warnings   True | False. Defaults to False. If set to True,
+                        then for succeeding commands, stderr is cleared
+
+    """
+    discard_warnings = kwargs.pop('discard_warnings', False)
+
+    try:
+        out, err = execute(*args, **kwargs)
+        failed = False
+    except ProcessExecutionError as exn:
+        out, err = '', six.text_type(exn)
+        failed = True
+
+    if not failed and discard_warnings and err:
+        # Handle commands that output to stderr but otherwise succeed
+        err = ''
+
+    return out, err
+
+
+def ssh_execute(ssh, cmd, process_input=None,
+                addl_env=None, check_exit_code=True):
+    sanitized_cmd = strutils.mask_password(cmd)
+    LOG.debug('Running cmd (SSH): %s', sanitized_cmd)
+    if addl_env:
+        raise InvalidArgumentError(_('Environment not supported over SSH'))
+
+    if process_input:
+        # This is (probably) fixable if we need it...
+        raise InvalidArgumentError(_('process_input not supported over SSH'))
+
+    stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
+    channel = stdout_stream.channel
+
+    # NOTE(justinsb): This seems suspicious...
+    # ...other SSH clients have buffering issues with this approach
+    stdout = stdout_stream.read()
+    sanitized_stdout = strutils.mask_password(stdout)
+    stderr = stderr_stream.read()
+    sanitized_stderr = strutils.mask_password(stderr)
+
+    stdin_stream.close()
+
+    exit_status = channel.recv_exit_status()
+
+    # exit_status == -1 if no exit code was returned
+    if exit_status != -1:
+        LOG.debug('Result was %s' % exit_status)
+        if check_exit_code and exit_status != 0:
+            raise ProcessExecutionError(exit_code=exit_status,
+                                        stdout=sanitized_stdout,
+                                        stderr=sanitized_stderr,
+                                        cmd=sanitized_cmd)
+
+    return (sanitized_stdout, sanitized_stderr)
+
+
+def get_worker_count():
+    """Utility to get the default worker count.
+
+    @return: The number of CPUs if that can be determined, else a default
+             worker count of 1 is returned.
+    """
+    try:
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        return 1

nova/service.py

@@ -21,7 +21,6 @@ import os
 import random
 import sys
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo import messaging
 from oslo.utils import importutils
@@ -34,6 +33,7 @@ from nova import exception
 from nova.i18n import _, _LE
 from nova.objects import base as objects_base
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova.openstack.common import service
 from nova import rpc
 from nova import servicegroup

nova/storage/linuxscsi.py

@@ -14,11 +14,10 @@
 
 """Generic linux scsi subsystem utilities."""
 
-from oslo.concurrency import processutils
-
 from nova.i18n import _LW
 from nova.openstack.common import log as logging
 from nova.openstack.common import loopingcall
+from nova.openstack.common import processutils
 from nova import utils
 
 LOG = logging.getLogger(__name__)

nova/tests/db/test_migrations.py

@@ -44,7 +44,6 @@ import glob
 import os
 
 from migrate.versioning import repository
-from oslo.concurrency import processutils
 from oslo.db.sqlalchemy import session
 from oslo.db.sqlalchemy import utils as oslodbutils
 import six.moves.urllib.parse as urlparse
@@ -55,6 +54,7 @@ import nova.db.sqlalchemy.migrate_repo
 from nova.db.sqlalchemy import utils as db_utils
 from nova.i18n import _
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 

nova/tests/fake_processutils.py

@@ -12,15 +12,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""This modules stubs out functions in oslo.concurrency.processutils."""
+"""This modules stubs out functions in openstack.common.processutils."""
 
 import re
 
 from eventlet import greenthread
-from oslo.concurrency import processutils
 import six
 
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 
 LOG = logging.getLogger(__name__)
 

nova/tests/network/test_linux_net.py

@@ -20,7 +20,6 @@ import os
 
 import mock
 import mox
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 from oslo.utils import timeutils
@@ -33,6 +32,7 @@ from nova.network import linux_net
 from nova import objects
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 

nova/tests/network/test_manager.py

@@ -21,7 +21,6 @@ import fixtures
 import mock
 import mox
 import netaddr
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.db import exception as db_exc
 from oslo import messaging
@@ -41,6 +40,7 @@ from nova import objects
 from nova.objects import quotas as quotas_obj
 from nova.objects import virtual_interface as vif_obj
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import quota
 from nova import test
 from nova.tests import fake_instance

nova/tests/network/test_neutronv2.py

@@ -2671,7 +2671,7 @@ class TestNeutronv2WithMock(test.TestCase):
             'fake-user', 'fake-project',
             auth_token='bff4a5a6b9eb4ea2a6efec6eefb77936')
 
-    @mock.patch('oslo.concurrency.lockutils.lock')
+    @mock.patch('nova.openstack.common.lockutils.lock')
     def test_get_instance_nw_info_locks_per_instance(self, mock_lock):
         instance = objects.Instance(uuid=uuid.uuid4())
         api = neutronapi.API()

nova/tests/test_crypto.py

@@ -20,11 +20,11 @@ import os
 
 import mock
 import mox
-from oslo.concurrency import processutils
 
 from nova import crypto
 from nova import db
 from nova import exception
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 

nova/tests/test_service.py

@@ -22,7 +22,6 @@ import sys
 
 import mock
 import mox
-from oslo.concurrency import processutils
 from oslo.config import cfg
 import testtools
 
@@ -30,6 +29,7 @@ from nova import context
 from nova import db
 from nova import exception
 from nova import manager
+from nova.openstack.common import processutils
 from nova.openstack.common import service as _service
 from nova import rpc
 from nova import service

nova/tests/test_utils.py

@@ -24,12 +24,12 @@ import tempfile
 
 import mox
 import netaddr
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import timeutils
 
 import nova
 from nova import exception
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 

nova/tests/virt/disk/test_api.py

@@ -16,8 +16,8 @@
 import tempfile
 
 import fixtures
-from oslo.concurrency import processutils
 
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 from nova.virt.disk import api

nova/tests/virt/disk/vfs/test_localfs.py

@@ -12,10 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import exception
+from nova.openstack.common import processutils
 from nova import test
 from nova.tests import utils as tests_utils
 import nova.utils

nova/tests/virt/libvirt/test_driver.py

@@ -32,8 +32,6 @@ import fixtures
 from lxml import etree
 import mock
 import mox
-from oslo.concurrency import lockutils
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 from oslo.utils import importutils
@@ -55,7 +53,9 @@ from nova import exception
 from nova.network import model as network_model
 from nova import objects
 from nova.openstack.common import fileutils
+from nova.openstack.common import lockutils
 from nova.openstack.common import loopingcall
+from nova.openstack.common import processutils
 from nova.openstack.common import uuidutils
 from nova.pci import manager as pci_manager
 from nova import test

nova/tests/virt/libvirt/test_firewall.py

@@ -21,12 +21,12 @@ from xml.dom import minidom
 from lxml import etree
 import mock
 import mox
-from oslo.concurrency import lockutils
 
 from nova.compute import utils as compute_utils
 from nova import exception
 from nova.network import linux_net
 from nova import objects
+from nova.openstack.common import lockutils
 from nova import test
 from nova.tests import fake_network
 from nova.tests.virt.libvirt import fakelibvirt

nova/tests/virt/libvirt/test_imagecache.py

@@ -20,7 +20,6 @@ import hashlib
 import os
 import time
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 from oslo.utils import importutils
@@ -28,6 +27,7 @@ from oslo.utils import importutils
 from nova import conductor
 from nova import db
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import test
 from nova.tests import fake_instance
 from nova import utils

nova/tests/virt/libvirt/test_lvm.py

@@ -16,10 +16,10 @@
 import contextlib
 
 import mock
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import exception
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 from nova.virt.libvirt import lvm

nova/tests/virt/libvirt/test_utils.py

@@ -18,11 +18,11 @@ import os
 import tempfile
 
 import mock
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import exception
 from nova.openstack.common import fileutils
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 from nova.virt.disk import api as disk

nova/tests/virt/libvirt/test_vif.py

@@ -16,12 +16,12 @@ import contextlib
 
 from lxml import etree
 import mock
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import exception
 from nova.network import linux_net
 from nova.network import model as network_model
+from nova.openstack.common import processutils
 from nova import test
 from nova.tests.virt.libvirt import fakelibvirt
 from nova import utils

nova/tests/virt/libvirt/test_volume.py

@@ -19,10 +19,10 @@ import time
 
 import fixtures
 import mock
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import exception
+from nova.openstack.common import processutils
 from nova.storage import linuxscsi
 from nova import test
 from nova.tests.virt.libvirt import fake_libvirt_utils

nova/tests/virt/test_images.py

@@ -15,9 +15,9 @@
 import os
 
 import mock
-from oslo.concurrency import processutils
 
 from nova import exception
+from nova.openstack.common import processutils
 from nova import test
 from nova import utils
 from nova.virt import images

nova/tests/virt/xenapi/test_vm_utils.py

@@ -20,7 +20,6 @@ from eventlet import greenthread
 import fixtures
 import mock
 import mox
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import timeutils
 from oslo.utils import units
@@ -32,6 +31,7 @@ from nova.compute import vm_mode
 from nova import context
 from nova import exception
 from nova.i18n import _
+from nova.openstack.common import processutils
 from nova import test
 from nova.tests.virt.xenapi import stubs
 from nova.tests.virt.xenapi import test_xenapi

nova/utils.py

@@ -36,8 +36,6 @@ from xml.sax import saxutils
 
 import eventlet
 import netaddr
-from oslo.concurrency import lockutils
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo import messaging
 from oslo.utils import excutils
@@ -47,7 +45,9 @@ import six
 
 from nova import exception
 from nova.i18n import _, _LE
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 
 notify_decorator = 'nova.notifications.notify_decorator'
 

nova/virt/disk/api.py

@@ -30,7 +30,6 @@ import tempfile
 if os.name != 'nt':
     import crypt
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 
@@ -39,6 +38,7 @@ from nova.i18n import _
 from nova.i18n import _LE
 from nova.i18n import _LW
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 from nova.virt.disk.mount import api as mount
 from nova.virt.disk.vfs import api as vfs

nova/virt/hyperv/vmops.py

@@ -22,7 +22,6 @@ import os
 import time
 
 from eventlet import timeout as etimeout
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import excutils
 from oslo.utils import importutils
@@ -34,6 +33,7 @@ from nova.i18n import _, _LI, _LE, _LW
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import loopingcall
+from nova.openstack.common import processutils
 from nova.openstack.common import uuidutils
 from nova import utils
 from nova.virt import configdrive

nova/virt/libvirt/driver.py

@@ -48,7 +48,6 @@ from eventlet import patcher
 from eventlet import tpool
 from eventlet import util as eventlet_util
 from lxml import etree
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 from oslo.utils import excutils
@@ -81,6 +80,7 @@ from nova import objects
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import loopingcall
+from nova.openstack.common import processutils
 from nova.pci import manager as pci_manager
 from nova.pci import utils as pci_utils
 from nova.pci import whitelist as pci_whitelist

nova/virt/libvirt/imagecache.py

@@ -25,7 +25,6 @@ import os
 import re
 import time
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.serialization import jsonutils
 
@@ -34,6 +33,7 @@ from nova.i18n import _LI
 from nova.i18n import _LW
 from nova.openstack.common import fileutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 from nova.virt import imagecache
 from nova.virt.libvirt import utils as libvirt_utils

nova/virt/libvirt/lvm.py

@@ -21,7 +21,6 @@
 
 import functools
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import units
 import six
@@ -31,6 +30,7 @@ from nova.i18n import _
 from nova.i18n import _LE
 from nova.i18n import _LW
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils as nova_utils
 from nova.virt.libvirt import utils
 

nova/virt/libvirt/utils.py

@@ -24,13 +24,13 @@ import platform
 import re
 
 from lxml import etree
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova.i18n import _
 from nova.i18n import _LI
 from nova.i18n import _LW
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 from nova.virt import images
 from nova.virt.libvirt import config as vconfig

nova/virt/libvirt/vif.py

@@ -19,7 +19,6 @@
 
 import copy
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 
 from nova import exception
@@ -28,6 +27,7 @@ from nova.i18n import _LE
 from nova.network import linux_net
 from nova.network import model as network_model
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 from nova.virt.libvirt import config as vconfig
 from nova.virt.libvirt import designer

nova/virt/libvirt/volume.py

@@ -21,7 +21,6 @@ import os
 import time
 import urllib2
 
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import strutils
 import six
@@ -33,6 +32,7 @@ from nova.i18n import _LE
 from nova.i18n import _LW
 from nova.openstack.common import log as logging
 from nova.openstack.common import loopingcall
+from nova.openstack.common import processutils
 from nova import paths
 from nova.storage import linuxscsi
 from nova import utils

nova/virt/vmwareapi/imagecache.py

@@ -35,12 +35,12 @@ the timestamp folder will be locked  and the timestamps will be purged.
 This will ensure that a image is not deleted during the spawn.
 """
 
-from oslo.concurrency import lockutils
 from oslo.config import cfg
 from oslo.utils import timeutils
 from oslo.vmware import exceptions as vexc
 
 from nova.i18n import _
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova.virt import imagecache
 from nova.virt.vmwareapi import ds_util

nova/virt/vmwareapi/vmops.py

@@ -24,7 +24,6 @@ import os
 import time
 
 import decorator
-from oslo.concurrency import lockutils
 from oslo.config import cfg
 from oslo.utils import excutils
 from oslo.utils import strutils
@@ -41,6 +40,7 @@ from nova import context as nova_context
 from nova import exception
 from nova.i18n import _, _LE, _LW
 from nova import objects
+from nova.openstack.common import lockutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import uuidutils
 from nova import utils

nova/virt/xenapi/vm_utils.py

@@ -28,7 +28,6 @@ from xml.dom import minidom
 from xml.parsers import expat
 
 from eventlet import greenthread
-from oslo.concurrency import processutils
 from oslo.config import cfg
 from oslo.utils import excutils
 from oslo.utils import importutils
@@ -47,6 +46,7 @@ from nova import exception
 from nova.i18n import _, _LE, _LI
 from nova.network import model as network_model
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova.openstack.common import versionutils
 from nova import utils
 from nova.virt import configdrive

nova/volume/encryptors/luks.py

@@ -16,9 +16,8 @@
 
 import re
 
-from oslo.concurrency import processutils
-
 from nova.openstack.common import log as logging
+from nova.openstack.common import processutils
 from nova import utils
 from nova.volume.encryptors import cryptsetup
 

openstack-common.conf

@@ -13,6 +13,7 @@ module=fixture.moxstubout
 module=imageutils
 module=install_venv_common
 module=local
+module=lockutils
 module=log
 module=loopingcall
 module=memorycache
@@ -20,6 +21,7 @@ module=middleware.base
 module=middleware.request_id
 module=periodic_task
 module=policy
+module=processutils
 module=report
 module=report.generators
 module=report.models

requirements.txt

@@ -34,7 +34,6 @@ six>=1.7.0
 stevedore>=1.0.0  # Apache-2.0
 websockify>=0.6.0,<0.7
 wsgiref>=0.1.2
-oslo.concurrency>=0.1.0  # Apache-2.0
 oslo.config>=1.4.0  # Apache-2.0
 oslo.serialization>=1.0.0               # Apache-2.0
 oslo.utils>=1.0.0                       # Apache-2.0

run_tests.sh

@@ -117,7 +117,7 @@ function run_tests {
       # provided.
       testrargs="discover ./nova/tests"
     fi
-    ${wrapper} python -m oslo.concurrency.lockutils python -m testtools.run $testropts $testrargs
+    ${wrapper} python -m nova.openstack.common.lockutils python -m testtools.run $testropts $testrargs
 
     # Short circuit because all of the testr and coverage stuff
     # below does not make sense when running testtools.run for
@@ -184,7 +184,7 @@ function run_pep8 {
 }
 
 
-TESTRTESTS="python -m oslo.concurrency.lockutils python setup.py testr"
+TESTRTESTS="python -m nova.openstack.common.lockutils python setup.py testr"
 
 if [ $never_venv -eq 0 ]
 then

tools/pretty_tox.sh

@@ -3,4 +3,4 @@
 set -o pipefail
 
 TESTRARGS=$1
-python -m oslo.concurrency.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit-trace.py -f
+python -m nova.openstack.common.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | $(dirname $0)/subunit-trace.py -f

tox.ini

@@ -40,7 +40,7 @@ commands = bash tools/lintstack.sh
 # tests conflict with coverage.
 commands =
   coverage erase
-  python -m oslo.concurrency.lockutils python setup.py testr --coverage \
+  python -m nova.openstack.common.lockutils python setup.py testr --coverage \
     --testr-args='{posargs}'
   coverage combine
   coverage html --include='nova/*' --omit='nova/openstack/common/*' -d covhtml -i