Boris Pavlovic
e305dad557
To be able to create profiling traces for Glance, client should be able to send special HTTP header that contains trace info. This patch is as well important to be able to make cross project traces. (Typical case nova calls glance via python client, if profiler is initialized in nova, glance client will add extra header, that will be parsed by special osprofiler middleware in glance api) Don't worry no security issue here, trace information is signed by HMAC key that is setted in api-paste.ini. So only person that knows HMAC key is able to send proper header. Change-Id: Ib13084fbe9b33c2f3dee165f7d6c778546cce6ca
555 lines
20 KiB
Python
555 lines
20 KiB
Python
# Copyright 2012 OpenStack Foundation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import copy
|
|
import errno
|
|
import hashlib
|
|
import logging
|
|
import posixpath
|
|
import socket
|
|
import struct
|
|
|
|
import six
|
|
from six.moves import http_client
|
|
from six.moves.urllib import parse
|
|
|
|
try:
|
|
import json
|
|
except ImportError:
|
|
import simplejson as json
|
|
|
|
# Python 2.5 compat fix
|
|
if not hasattr(parse, 'parse_qsl'):
|
|
import cgi
|
|
parse.parse_qsl = cgi.parse_qsl
|
|
|
|
import OpenSSL
|
|
|
|
from glanceclient.common import utils
|
|
from glanceclient import exc
|
|
from glanceclient.openstack.common import importutils
|
|
from glanceclient.openstack.common import network_utils
|
|
from glanceclient.openstack.common import strutils
|
|
|
|
osprofiler_web = importutils.try_import("osprofiler.web")
|
|
|
|
try:
|
|
from eventlet import patcher
|
|
# Handle case where we are running in a monkey patched environment
|
|
if patcher.is_monkey_patched('socket'):
|
|
from eventlet.green.httplib import HTTPSConnection
|
|
from eventlet.green.OpenSSL.SSL import GreenConnection as Connection
|
|
from eventlet.greenio import GreenSocket
|
|
# TODO(mclaren): A getsockopt workaround: see 'getsockopt' doc string
|
|
GreenSocket.getsockopt = utils.getsockopt
|
|
else:
|
|
raise ImportError
|
|
except ImportError:
|
|
HTTPSConnection = http_client.HTTPSConnection
|
|
from OpenSSL.SSL import Connection as Connection
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
USER_AGENT = 'python-glanceclient'
|
|
CHUNKSIZE = 1024 * 64 # 64kB
|
|
|
|
|
|
class HTTPClient(object):
|
|
|
|
def __init__(self, endpoint, **kwargs):
|
|
self.endpoint = endpoint
|
|
endpoint_parts = self.parse_endpoint(self.endpoint)
|
|
self.endpoint_scheme = endpoint_parts.scheme
|
|
self.endpoint_hostname = endpoint_parts.hostname
|
|
self.endpoint_port = endpoint_parts.port
|
|
self.endpoint_path = endpoint_parts.path
|
|
|
|
self.connection_class = self.get_connection_class(self.endpoint_scheme)
|
|
self.connection_kwargs = self.get_connection_kwargs(
|
|
self.endpoint_scheme, **kwargs)
|
|
|
|
self.identity_headers = kwargs.get('identity_headers')
|
|
self.auth_token = kwargs.get('token')
|
|
if self.identity_headers:
|
|
if self.identity_headers.get('X-Auth-Token'):
|
|
self.auth_token = self.identity_headers.get('X-Auth-Token')
|
|
del self.identity_headers['X-Auth-Token']
|
|
|
|
@staticmethod
|
|
def parse_endpoint(endpoint):
|
|
return network_utils.urlsplit(endpoint)
|
|
|
|
@staticmethod
|
|
def get_connection_class(scheme):
|
|
if scheme == 'https':
|
|
return VerifiedHTTPSConnection
|
|
else:
|
|
return http_client.HTTPConnection
|
|
|
|
@staticmethod
|
|
def get_connection_kwargs(scheme, **kwargs):
|
|
_kwargs = {'timeout': float(kwargs.get('timeout', 600))}
|
|
|
|
if scheme == 'https':
|
|
_kwargs['cacert'] = kwargs.get('cacert', None)
|
|
_kwargs['cert_file'] = kwargs.get('cert_file', None)
|
|
_kwargs['key_file'] = kwargs.get('key_file', None)
|
|
_kwargs['insecure'] = kwargs.get('insecure', False)
|
|
_kwargs['ssl_compression'] = kwargs.get('ssl_compression', True)
|
|
|
|
return _kwargs
|
|
|
|
def get_connection(self):
|
|
_class = self.connection_class
|
|
try:
|
|
return _class(self.endpoint_hostname, self.endpoint_port,
|
|
**self.connection_kwargs)
|
|
except http_client.InvalidURL:
|
|
raise exc.InvalidEndpoint()
|
|
|
|
def log_curl_request(self, method, url, kwargs):
|
|
curl = ['curl -i -X %s' % method]
|
|
|
|
for (key, value) in kwargs['headers'].items():
|
|
header = '-H \'%s: %s\'' % (key, value)
|
|
curl.append(header)
|
|
|
|
conn_params_fmt = [
|
|
('key_file', '--key %s'),
|
|
('cert_file', '--cert %s'),
|
|
('cacert', '--cacert %s'),
|
|
]
|
|
for (key, fmt) in conn_params_fmt:
|
|
value = self.connection_kwargs.get(key)
|
|
if value:
|
|
curl.append(fmt % value)
|
|
|
|
if self.connection_kwargs.get('insecure'):
|
|
curl.append('-k')
|
|
|
|
if kwargs.get('body') is not None:
|
|
curl.append('-d \'%s\'' % kwargs['body'])
|
|
|
|
curl.append('%s%s' % (self.endpoint, url))
|
|
LOG.debug(strutils.safe_encode(' '.join(curl)))
|
|
|
|
@staticmethod
|
|
def log_http_response(resp, body=None):
|
|
status = (resp.version / 10.0, resp.status, resp.reason)
|
|
dump = ['\nHTTP/%.1f %s %s' % status]
|
|
dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()])
|
|
dump.append('')
|
|
if body:
|
|
dump.extend([body, ''])
|
|
LOG.debug(strutils.safe_encode('\n'.join(dump)))
|
|
|
|
@staticmethod
|
|
def encode_headers(headers):
|
|
"""Encodes headers.
|
|
|
|
Note: This should be used right before
|
|
sending anything out.
|
|
|
|
:param headers: Headers to encode
|
|
:returns: Dictionary with encoded headers'
|
|
names and values
|
|
"""
|
|
return dict((strutils.safe_encode(h), strutils.safe_encode(v))
|
|
for h, v in six.iteritems(headers))
|
|
|
|
def _http_request(self, url, method, **kwargs):
|
|
"""Send an http request with the specified characteristics.
|
|
|
|
Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
|
|
as setting headers and error handling.
|
|
"""
|
|
# Copy the kwargs so we can reuse the original in case of redirects
|
|
kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {}))
|
|
kwargs['headers'].setdefault('User-Agent', USER_AGENT)
|
|
|
|
if osprofiler_web:
|
|
kwargs['headers'].update(osprofiler_web.get_trace_id_headers())
|
|
|
|
if self.auth_token:
|
|
kwargs['headers'].setdefault('X-Auth-Token', self.auth_token)
|
|
|
|
if self.identity_headers:
|
|
for k, v in six.iteritems(self.identity_headers):
|
|
kwargs['headers'].setdefault(k, v)
|
|
|
|
self.log_curl_request(method, url, kwargs)
|
|
conn = self.get_connection()
|
|
|
|
# Note(flaper87): Before letting headers / url fly,
|
|
# they should be encoded otherwise httplib will
|
|
# complain. If we decide to rely on python-request
|
|
# this wont be necessary anymore.
|
|
kwargs['headers'] = self.encode_headers(kwargs['headers'])
|
|
|
|
try:
|
|
if self.endpoint_path:
|
|
# NOTE(yuyangbj): this method _http_request could either be
|
|
# called by API layer, or be called recursively with
|
|
# redirection. For example, url would be '/v1/images/detail'
|
|
# from API layer, but url would be 'https://example.com:92/
|
|
# v1/images/detail' from recursion.
|
|
# See bug #1230032 and bug #1208618.
|
|
if url is not None:
|
|
all_parts = parse.urlparse(url)
|
|
if not (all_parts.scheme and all_parts.netloc):
|
|
norm_parse = posixpath.normpath
|
|
url = norm_parse('/'.join([self.endpoint_path, url]))
|
|
else:
|
|
url = self.endpoint_path
|
|
|
|
conn_url = parse.urlsplit(url).geturl()
|
|
# Note(flaper87): Ditto, headers / url
|
|
# encoding to make httplib happy.
|
|
conn_url = strutils.safe_encode(conn_url)
|
|
if kwargs['headers'].get('Transfer-Encoding') == 'chunked':
|
|
conn.putrequest(method, conn_url)
|
|
for header, value in kwargs['headers'].items():
|
|
conn.putheader(header, value)
|
|
conn.endheaders()
|
|
chunk = kwargs['body'].read(CHUNKSIZE)
|
|
# Chunk it, baby...
|
|
while chunk:
|
|
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
|
|
chunk = kwargs['body'].read(CHUNKSIZE)
|
|
conn.send('0\r\n\r\n')
|
|
else:
|
|
conn.request(method, conn_url, **kwargs)
|
|
resp = conn.getresponse()
|
|
except socket.gaierror as e:
|
|
message = "Error finding address for %s: %s" % (
|
|
self.endpoint_hostname, e)
|
|
raise exc.InvalidEndpoint(message=message)
|
|
except (socket.error, socket.timeout) as e:
|
|
endpoint = self.endpoint
|
|
message = ("Error communicating with %(endpoint)s %(e)s" %
|
|
{'endpoint': endpoint, 'e': e})
|
|
raise exc.CommunicationError(message=message)
|
|
|
|
body_iter = ResponseBodyIterator(resp)
|
|
|
|
# Read body into string if it isn't obviously image data
|
|
if resp.getheader('content-type', None) != 'application/octet-stream':
|
|
body_str = ''.join([chunk for chunk in body_iter])
|
|
self.log_http_response(resp, body_str)
|
|
body_iter = six.StringIO(body_str)
|
|
else:
|
|
self.log_http_response(resp)
|
|
|
|
if 400 <= resp.status < 600:
|
|
LOG.debug("Request returned failure status: %d" % resp.status)
|
|
raise exc.from_response(resp, body_str)
|
|
elif resp.status in (301, 302, 305):
|
|
# Redirected. Reissue the request to the new location.
|
|
return self._http_request(resp.getheader('location', None), method,
|
|
**kwargs)
|
|
elif resp.status == 300:
|
|
raise exc.from_response(resp)
|
|
|
|
return resp, body_iter
|
|
|
|
def json_request(self, method, url, **kwargs):
|
|
kwargs.setdefault('headers', {})
|
|
kwargs['headers'].setdefault('Content-Type', 'application/json')
|
|
|
|
if 'body' in kwargs:
|
|
kwargs['body'] = json.dumps(kwargs['body'])
|
|
|
|
resp, body_iter = self._http_request(url, method, **kwargs)
|
|
|
|
if 'application/json' in resp.getheader('content-type', ''):
|
|
body = ''.join([chunk for chunk in body_iter])
|
|
try:
|
|
body = json.loads(body)
|
|
except ValueError:
|
|
LOG.error('Could not decode response body as JSON')
|
|
else:
|
|
body = None
|
|
|
|
return resp, body
|
|
|
|
def raw_request(self, method, url, **kwargs):
|
|
kwargs.setdefault('headers', {})
|
|
kwargs['headers'].setdefault('Content-Type',
|
|
'application/octet-stream')
|
|
|
|
if 'content_length' in kwargs:
|
|
content_length = kwargs.pop('content_length')
|
|
else:
|
|
content_length = None
|
|
|
|
if (('body' in kwargs) and (hasattr(kwargs['body'], 'read') and
|
|
method.lower() in ('post', 'put'))):
|
|
|
|
# NOTE(dosaboy): only use chunked transfer if not setting a
|
|
# content length since setting it will implicitly disable
|
|
# chunking.
|
|
|
|
file_content_length = utils.get_file_size(kwargs['body'])
|
|
if content_length is None:
|
|
content_length = file_content_length
|
|
elif (file_content_length and
|
|
(content_length != file_content_length)):
|
|
errmsg = ("supplied content-length (%s) does not match "
|
|
"length of supplied data (%s)" %
|
|
(content_length, file_content_length))
|
|
raise AttributeError(errmsg)
|
|
|
|
if content_length is None:
|
|
# We use 'Transfer-Encoding: chunked' because
|
|
# body size may not always be known in advance.
|
|
kwargs['headers']['Transfer-Encoding'] = 'chunked'
|
|
else:
|
|
kwargs['headers']['Content-Length'] = str(content_length)
|
|
|
|
return self._http_request(url, method, **kwargs)
|
|
|
|
|
|
class OpenSSLConnectionDelegator(object):
|
|
"""
|
|
An OpenSSL.SSL.Connection delegator.
|
|
|
|
Supplies an additional 'makefile' method which httplib requires
|
|
and is not present in OpenSSL.SSL.Connection.
|
|
|
|
Note: Since it is not possible to inherit from OpenSSL.SSL.Connection
|
|
a delegator must be used.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
self.connection = Connection(*args, **kwargs)
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.connection, name)
|
|
|
|
def makefile(self, *args, **kwargs):
|
|
# Making sure socket is closed when this file is closed
|
|
# since we now avoid closing socket on connection close
|
|
# see new close method under VerifiedHTTPSConnection
|
|
kwargs['close'] = True
|
|
|
|
return socket._fileobject(self.connection, *args, **kwargs)
|
|
|
|
|
|
class VerifiedHTTPSConnection(HTTPSConnection):
|
|
"""
|
|
Extended HTTPSConnection which uses the OpenSSL library
|
|
for enhanced SSL support.
|
|
Note: Much of this functionality can eventually be replaced
|
|
with native Python 3.3 code.
|
|
"""
|
|
def __init__(self, host, port=None, key_file=None, cert_file=None,
|
|
cacert=None, timeout=None, insecure=False,
|
|
ssl_compression=True):
|
|
HTTPSConnection.__init__(self, host, port,
|
|
key_file=key_file,
|
|
cert_file=cert_file)
|
|
self.key_file = key_file
|
|
self.cert_file = cert_file
|
|
self.timeout = timeout
|
|
self.insecure = insecure
|
|
self.ssl_compression = ssl_compression
|
|
self.cacert = cacert
|
|
self.setcontext()
|
|
|
|
@staticmethod
|
|
def host_matches_cert(host, x509):
|
|
"""
|
|
Verify that the the x509 certificate we have received
|
|
from 'host' correctly identifies the server we are
|
|
connecting to, ie that the certificate's Common Name
|
|
or a Subject Alternative Name matches 'host'.
|
|
"""
|
|
def check_match(name):
|
|
# Directly match the name
|
|
if name == host:
|
|
return True
|
|
|
|
# Support single wildcard matching
|
|
if name.startswith('*.') and host.find('.') > 0:
|
|
if name[2:] == host.split('.', 1)[1]:
|
|
return True
|
|
|
|
common_name = x509.get_subject().commonName
|
|
|
|
# First see if we can match the CN
|
|
if check_match(common_name):
|
|
return True
|
|
|
|
# Also try Subject Alternative Names for a match
|
|
san_list = None
|
|
for i in range(x509.get_extension_count()):
|
|
ext = x509.get_extension(i)
|
|
if ext.get_short_name() == 'subjectAltName':
|
|
san_list = str(ext)
|
|
for san in ''.join(san_list.split()).split(','):
|
|
if san.startswith('DNS:'):
|
|
if check_match(san.split(':', 1)[1]):
|
|
return True
|
|
|
|
# Server certificate does not match host
|
|
msg = ('Host "%s" does not match x509 certificate contents: '
|
|
'CommonName "%s"' % (host, common_name))
|
|
if san_list is not None:
|
|
msg = msg + ', subjectAltName "%s"' % san_list
|
|
raise exc.SSLCertificateError(msg)
|
|
|
|
def verify_callback(self, connection, x509, errnum,
|
|
depth, preverify_ok):
|
|
# NOTE(leaman): preverify_ok may be a non-boolean type
|
|
preverify_ok = bool(preverify_ok)
|
|
if x509.has_expired():
|
|
msg = "SSL Certificate expired on '%s'" % x509.get_notAfter()
|
|
raise exc.SSLCertificateError(msg)
|
|
|
|
if depth == 0 and preverify_ok:
|
|
# We verify that the host matches against the last
|
|
# certificate in the chain
|
|
return self.host_matches_cert(self.host, x509)
|
|
else:
|
|
# Pass through OpenSSL's default result
|
|
return preverify_ok
|
|
|
|
def setcontext(self):
|
|
"""
|
|
Set up the OpenSSL context.
|
|
"""
|
|
self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
|
|
|
|
if self.ssl_compression is False:
|
|
self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION
|
|
|
|
if self.insecure is not True:
|
|
self.context.set_verify(OpenSSL.SSL.VERIFY_PEER,
|
|
self.verify_callback)
|
|
else:
|
|
self.context.set_verify(OpenSSL.SSL.VERIFY_NONE,
|
|
lambda *args: True)
|
|
|
|
if self.cert_file:
|
|
try:
|
|
self.context.use_certificate_file(self.cert_file)
|
|
except Exception as e:
|
|
msg = 'Unable to load cert from "%s" %s' % (self.cert_file, e)
|
|
raise exc.SSLConfigurationError(msg)
|
|
if self.key_file is None:
|
|
# We support having key and cert in same file
|
|
try:
|
|
self.context.use_privatekey_file(self.cert_file)
|
|
except Exception as e:
|
|
msg = ('No key file specified and unable to load key '
|
|
'from "%s" %s' % (self.cert_file, e))
|
|
raise exc.SSLConfigurationError(msg)
|
|
|
|
if self.key_file:
|
|
try:
|
|
self.context.use_privatekey_file(self.key_file)
|
|
except Exception as e:
|
|
msg = 'Unable to load key from "%s" %s' % (self.key_file, e)
|
|
raise exc.SSLConfigurationError(msg)
|
|
|
|
if self.cacert:
|
|
try:
|
|
self.context.load_verify_locations(self.cacert)
|
|
except Exception as e:
|
|
msg = ('Unable to load CA from "%(cacert)s" %(exc)s' %
|
|
dict(cacert=self.cacert, exc=e))
|
|
raise exc.SSLConfigurationError(msg)
|
|
else:
|
|
self.context.set_default_verify_paths()
|
|
|
|
def connect(self):
|
|
"""
|
|
Connect to an SSL port using the OpenSSL library and apply
|
|
per-connection parameters.
|
|
"""
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
if self.timeout is not None:
|
|
# '0' microseconds
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO,
|
|
struct.pack('fL', self.timeout, 0))
|
|
self.sock = OpenSSLConnectionDelegator(self.context, sock)
|
|
self.sock.connect((self.host, self.port))
|
|
|
|
def close(self):
|
|
if self.sock:
|
|
# Removing reference to socket but don't close it yet.
|
|
# Response close will close both socket and associated
|
|
# file. Closing socket too soon will cause response
|
|
# reads to fail with socket IO error 'Bad file descriptor'.
|
|
self.sock = None
|
|
|
|
# Calling close on HTTPConnection to continue doing that cleanup.
|
|
HTTPSConnection.close(self)
|
|
|
|
|
|
class ResponseBodyIterator(object):
|
|
"""
|
|
A class that acts as an iterator over an HTTP response.
|
|
|
|
This class will also check response body integrity when iterating over
|
|
the instance and if a checksum was supplied using `set_checksum` method,
|
|
else by default the class will not do any integrity check.
|
|
"""
|
|
|
|
def __init__(self, resp):
|
|
self._resp = resp
|
|
self._checksum = None
|
|
self._size = int(resp.getheader('content-length', 0))
|
|
self._end_reached = False
|
|
|
|
def set_checksum(self, checksum):
|
|
"""
|
|
Set checksum to check against when iterating over this instance.
|
|
|
|
:raise: AttributeError if iterator is already consumed.
|
|
"""
|
|
if self._end_reached:
|
|
raise AttributeError("Can't set checksum for an already consumed"
|
|
" iterator")
|
|
self._checksum = checksum
|
|
|
|
def __len__(self):
|
|
return int(self._size)
|
|
|
|
def __iter__(self):
|
|
md5sum = hashlib.md5()
|
|
while True:
|
|
try:
|
|
chunk = self.next()
|
|
except StopIteration:
|
|
self._end_reached = True
|
|
# NOTE(mouad): Check image integrity when the end of response
|
|
# body is reached.
|
|
md5sum = md5sum.hexdigest()
|
|
if self._checksum is not None and md5sum != self._checksum:
|
|
raise IOError(errno.EPIPE,
|
|
'Corrupted image. Checksum was %s '
|
|
'expected %s' % (md5sum, self._checksum))
|
|
raise
|
|
else:
|
|
yield chunk
|
|
md5sum.update(chunk)
|
|
|
|
def next(self):
|
|
chunk = self._resp.read(CHUNKSIZE)
|
|
if chunk:
|
|
return chunk
|
|
else:
|
|
raise StopIteration()
|