diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 29567f4543..66e361b5ce 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -157,6 +157,9 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, raise RetryableError(error) try: + validate_image_status_before_upload(conn, glance_host, glance_port, + image_id, url, extra_headers) + try: # NOTE(sirp): httplib under python2.4 won't accept # a file-like object to request @@ -227,60 +230,125 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, "Response Status: %i, Response body: %s" % (url, resp.status, resp.read())) - # Note(Jesse): This branch sorts errors into those that are permanent, - # those that are ephemeral, and those that are unexpected. - if resp.status in (httplib.BAD_REQUEST, # 400 - httplib.UNAUTHORIZED, # 401 - httplib.PAYMENT_REQUIRED, # 402 - httplib.FORBIDDEN, # 403 - httplib.NOT_FOUND, # 404 - httplib.METHOD_NOT_ALLOWED, # 405 - httplib.NOT_ACCEPTABLE, # 406 - httplib.PROXY_AUTHENTICATION_REQUIRED, # 407 - httplib.CONFLICT, # 409 - httplib.GONE, # 410 - httplib.LENGTH_REQUIRED, # 411 - httplib.PRECONDITION_FAILED, # 412 - httplib.REQUEST_ENTITY_TOO_LARGE, # 413 - httplib.REQUEST_URI_TOO_LONG, # 414 - httplib.UNSUPPORTED_MEDIA_TYPE, # 415 - httplib.REQUESTED_RANGE_NOT_SATISFIABLE, # 416 - httplib.EXPECTATION_FAILED, # 417 - httplib.UNPROCESSABLE_ENTITY, # 422 - httplib.LOCKED, # 423 - httplib.FAILED_DEPENDENCY, # 424 - httplib.UPGRADE_REQUIRED, # 426 - httplib.NOT_IMPLEMENTED, # 501 - httplib.HTTP_VERSION_NOT_SUPPORTED, # 505 - httplib.NOT_EXTENDED, # 510 - ): - raise PluginError("Got Permanent Error response [%i] while " - "uploading image [%s] to glance host [%s:%s]" - % (resp.status, image_id, - glance_host, glance_port)) - elif resp.status in (httplib.REQUEST_TIMEOUT, # 408 - httplib.INTERNAL_SERVER_ERROR, # 500 - httplib.BAD_GATEWAY, # 502 - httplib.SERVICE_UNAVAILABLE, # 503 - httplib.GATEWAY_TIMEOUT, # 504 - httplib.INSUFFICIENT_STORAGE, # 507 - ): - raise RetryableError("Got Ephemeral Error response [%i] while " - "uploading image [%s] to glance host [%s:%s]" - % (resp.status, image_id, - glance_host, glance_port)) - else: - # Note(Jesse): Assume unexpected errors are retryable. If you are - # seeing this error message, the error should probably be added - # to either the ephemeral or permanent error list. - raise RetryableError("Got Unexpected Error response [%i] while " - "uploading image [%s] to glance host [%s:%s]" - % (resp.status, image_id, - glance_host, glance_port)) + check_resp_status_and_retry(resp, image_id, glance_host, glance_port) + finally: conn.close() +def check_resp_status_and_retry(resp, image_id, glance_host, glance_port): + # Note(Jesse): This branch sorts errors into those that are permanent, + # those that are ephemeral, and those that are unexpected. + if resp.status in (httplib.BAD_REQUEST, # 400 + httplib.UNAUTHORIZED, # 401 + httplib.PAYMENT_REQUIRED, # 402 + httplib.FORBIDDEN, # 403 + httplib.NOT_FOUND, # 404 + httplib.METHOD_NOT_ALLOWED, # 405 + httplib.NOT_ACCEPTABLE, # 406 + httplib.PROXY_AUTHENTICATION_REQUIRED, # 407 + httplib.CONFLICT, # 409 + httplib.GONE, # 410 + httplib.LENGTH_REQUIRED, # 411 + httplib.PRECONDITION_FAILED, # 412 + httplib.REQUEST_ENTITY_TOO_LARGE, # 413 + httplib.REQUEST_URI_TOO_LONG, # 414 + httplib.UNSUPPORTED_MEDIA_TYPE, # 415 + httplib.REQUESTED_RANGE_NOT_SATISFIABLE, # 416 + httplib.EXPECTATION_FAILED, # 417 + httplib.UNPROCESSABLE_ENTITY, # 422 + httplib.LOCKED, # 423 + httplib.FAILED_DEPENDENCY, # 424 + httplib.UPGRADE_REQUIRED, # 426 + httplib.NOT_IMPLEMENTED, # 501 + httplib.HTTP_VERSION_NOT_SUPPORTED, # 505 + httplib.NOT_EXTENDED, # 510 + ): + raise PluginError("Got Permanent Error response [%i] while " + "uploading image [%s] to glance host [%s:%s]" + % (resp.status, image_id, + glance_host, glance_port)) + # NOTE(nikhil): Only a sub-set of the 500 errors are retryable. We + # optimistically retry on 500 errors below. + elif resp.status in (httplib.REQUEST_TIMEOUT, # 408 + httplib.INTERNAL_SERVER_ERROR, # 500 + httplib.BAD_GATEWAY, # 502 + httplib.SERVICE_UNAVAILABLE, # 503 + httplib.GATEWAY_TIMEOUT, # 504 + httplib.INSUFFICIENT_STORAGE, # 507 + ): + raise RetryableError("Got Ephemeral Error response [%i] while " + "uploading image [%s] to glance host [%s:%s]" + % (resp.status, image_id, + glance_host, glance_port)) + else: + # Note(Jesse): Assume unexpected errors are retryable. If you are + # seeing this error message, the error should probably be added + # to either the ephemeral or permanent error list. + raise RetryableError("Got Unexpected Error response [%i] while " + "uploading image [%s] to glance host [%s:%s]" + % (resp.status, image_id, + glance_host, glance_port)) + + +def validate_image_status_before_upload(conn, glance_host, glance_port, + image_id, url, extra_headers): + try: + # NOTE(nikhil): Attempt to determine if the Image has a status + # of 'queued'. Because data will continued to be sent to Glance + # until it has a chance to check the Image state, discover that + # it is not 'active' and send back a 409. Hence, the data will be + # unnecessarily buffered by Glance. This wastes time and bandwidth. + # LP bug #1202785 + conn.request('HEAD', '/v1/images/%s' % image_id, + headers=extra_headers) + head_resp = conn.getresponse() + # NOTE(nikhil): read the response to re-use the conn object. + body_data = head_resp.read(8192) + if len(body_data) > 8: + err_msg = ('Cannot upload data for image %(image_id)s as the ' + 'HEAD call had more than 8192 bytes of data in ' + 'the response body.' % {'image_id': image_id}) + raise PluginError("Got Permanent Error while uploading image " + "[%s] to glance host [%s:%s]. " + "Message: %s" % (image_id, glance_host, + glance_port, err_msg)) + else: + head_resp.read() + + except Exception, error: + logging.exception('Failed to HEAD the image %(image_id)s while ' + 'checking image status before attempting to ' + 'upload %(url)s' % {'image_id': image_id, + 'url': url}) + raise RetryableError(error) + + if head_resp.status != httplib.OK: + logging.error("Unexpected response while doing a HEAD call " + "to image %s , url = %s , Response Status: " + "%i" % (image_id, url, head_resp.status)) + + check_resp_status_and_retry(head_resp, image_id, glance_host, + glance_port) + + else: + image_status = head_resp.getheader('x-image-meta-status') + if image_status not in ('queued', ): + err_msg = ('Cannot upload data for image %(image_id)s as the ' + 'image status is %(image_status)s' % + {'image_id': image_id, 'image_status': image_status}) + logging.exception(err_msg) + raise PluginError("Got Permanent Error while uploading image " + "[%s] to glance host [%s:%s]. " + "Message: %s" % (image_id, glance_host, + glance_port, err_msg)) + else: + logging.info('Found image %(image_id)s in status ' + '%(image_status)s. Attempting to ' + 'upload.' % {'image_id': image_id, + 'image_status': image_status}) + + def download_vhd(session, image_id, glance_host, glance_port, glance_use_ssl, uuid_stack, sr_path, extra_headers): """Download an image from Glance, unbundle it, and then deposit the VHDs