Merge "Add support for passing image size to Glance API"

glanceclient/tests/unit/v2/base.py

@@ -87,6 +87,10 @@ class BaseController(testtools.TestCase):
         resp = self.controller.upload(*args, **kwargs)
         self._assertRequestId(resp)
 
+    def stage(self, *args, **kwargs):
+        resp = self.controller.stage(*args, **kwargs)
+        self._assertRequestId(resp)
+
     def data(self, *args, **kwargs):
         body = self.controller.data(*args, **kwargs)
         self._assertRequestId(body)

glanceclient/tests/unit/v2/test_images.py

@@ -195,6 +195,12 @@ data_fixtures = {
             '',
         ),
     },
+    '/v2/images/606b0e88-7c5a-4d54-b5bb-046105d4de6f/stage': {
+        'PUT': (
+            {},
+            '',
+        ),
+    },
     '/v2/images/5cc4bebc-db27-11e1-a1eb-080027cbe205/file': {
         'GET': (
             {},
@@ -1010,6 +1016,16 @@ class TestController(testtools.TestCase):
         self.assertEqual('3a4560a1-e585-443e-9b39-553b46ec92d1', image.id)
         self.assertEqual('image-1', image.name)
 
+    def test_create_image_w_size(self):
+        properties = {
+            'name': 'image-1',
+            'size': '4'
+        }
+        image = self.controller.create(**properties)
+        self.assertEqual('3a4560a1-e585-443e-9b39-553b46ec92d1', image.id)
+        self.assertEqual('image-1', image.name)
+        self.assertIsNone(image.get('size'))
+
     def test_create_bad_additionalProperty_type(self):
         properties = {
             'name': 'image-1',
@@ -1054,12 +1070,63 @@ class TestController(testtools.TestCase):
                   image_data)]
         self.assertEqual(expect, self.api.calls)
 
-    def test_data_upload_w_size(self):
+    def test_data_upload_with_invalid_size(self):
+        image_data = 'CCC'
+        image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f'
+        self.assertRaises(TypeError, self.controller.upload, image_id,
+                          image_data, image_size='invalid_size')
+        expect = []
+        self.assertEqual(expect, self.api.calls)
+
+    def test_data_upload_w_size_same_as_data(self):
         image_data = 'CCC'
         image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f'
         self.controller.upload(image_id, image_data, image_size=3)
         expect = [('PUT', '/v2/images/%s/file' % image_id,
-                  {'Content-Type': 'application/octet-stream'},
+                  {'Content-Type': 'application/octet-stream',
+                   'x-openstack-image-size': str(len(image_data))},
+                  image_data)]
+        self.assertEqual(expect, self.api.calls)
+
+    def test_data_upload_w_size_diff_than_data(self):
+        image_data = 'CCCCCC'
+        image_size = '3'
+        image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f'
+        self.controller.upload(image_id, image_data,
+                               image_size=int(image_size))
+        expect = [('PUT', '/v2/images/%s/file' % image_id,
+                  {'Content-Type': 'application/octet-stream',
+                   'x-openstack-image-size': image_size},
+                  image_data)]
+        self.assertEqual(expect, self.api.calls)
+
+    def test_data_stage_with_invalid_size(self):
+        image_data = 'CCC'
+        image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f'
+        self.assertRaises(TypeError, self.controller.stage, image_id,
+                          image_data, image_size='invalid_size')
+        expect = []
+        self.assertEqual(expect, self.api.calls)
+
+    def test_data_stage_w_size_same_as_data(self):
+        image_data = 'CCC'
+        image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f'
+        self.controller.stage(image_id, image_data, image_size=3)
+        expect = [('PUT', '/v2/images/%s/stage' % image_id,
+                  {'Content-Type': 'application/octet-stream',
+                   'x-openstack-image-size': str(len(image_data))},
+                  image_data)]
+        self.assertEqual(expect, self.api.calls)
+
+    def test_data_stage_w_size_diff_than_data(self):
+        image_data = 'CCCCCC'
+        image_size = '3'
+        image_id = '606b0e88-7c5a-4d54-b5bb-046105d4de6f'
+        self.controller.stage(image_id, image_data,
+                              image_size=int(image_size))
+        expect = [('PUT', '/v2/images/%s/stage' % image_id,
+                  {'Content-Type': 'application/octet-stream',
+                   'x-openstack-image-size': image_size},
                   image_data)]
         self.assertEqual(expect, self.api.calls)
 

glanceclient/tests/unit/v2/test_shell_v2.py

@@ -679,6 +679,117 @@ class ShellV2Test(testtools.TestCase):
                  {'quota': 'quota2', 'limit': 20, 'usage': 5}],
                 ['Quota', 'Limit', 'Usage'])
 
+    def test_do_image_stage_size_match(self):
+        args = mock.Mock()
+        args.id = 'IMG-01'
+        args.file = 'testfile'
+        args.size = 1024
+        args.progress = False
+
+        with mock.patch('glanceclient.common.utils.get_data_file',
+                        return_value='fileobj'), \
+                mock.patch('glanceclient.common.utils.get_file_size',
+                           return_value=1024), \
+                mock.patch('glanceclient.v2.shell._validate_backend'), \
+                mock.patch.object(self.gc.images,
+                                  'stage') as mock_stage:
+            test_shell.do_image_stage(self.gc, args)
+            mock_stage.assert_called_once_with('IMG-01', 'fileobj',
+                                               1024)
+
+    def test_do_image_stage_size_mismatch(self):
+        args = mock.Mock()
+        args.id = 'IMG-01'
+        args.file = 'testfile'
+        args.size = 1024
+        args.progress = False
+
+        with mock.patch('glanceclient.common.utils.get_data_file',
+                        return_value='fileobj'), \
+                mock.patch('glanceclient.common.utils.get_file_size',
+                           return_value=2048), \
+                mock.patch('glanceclient.v2.shell._validate_backend'):
+            with self.assertRaisesRegex(ValueError,
+                                        "Size mismatch: provided size 1024 "
+                                        "does not match the size of the "
+                                        "image 2048"):
+                test_shell.do_image_stage(self.gc, args)
+
+    def test_do_image_stage_no_size_in_args(self):
+        args = mock.Mock()
+        args.id = 'IMG-01'
+        args.file = 'testfile'
+        args.size = None
+        args.progress = False
+
+        with mock.patch('glanceclient.common.utils.get_data_file',
+                        return_value='fileobj'), \
+                mock.patch('glanceclient.common.utils.get_file_size',
+                           return_value=1024), \
+                mock.patch('glanceclient.v2.shell._validate_backend'), \
+                mock.patch.object(self.gc.images,
+                                  'stage') as mock_upload:
+            test_shell.do_image_stage(self.gc, args)
+            mock_upload.assert_called_once_with('IMG-01', 'fileobj',
+                                                1024)
+
+    def test_do_image_upload_size_match(self):
+        args = mock.Mock()
+        args.id = 'IMG-01'
+        args.file = 'testfile'
+        args.size = 1024
+        args.store = None
+        args.progress = False
+
+        with mock.patch('glanceclient.common.utils.get_data_file',
+                        return_value='fileobj'), \
+                mock.patch('glanceclient.common.utils.get_file_size',
+                           return_value=1024), \
+                mock.patch('glanceclient.v2.shell._validate_backend'), \
+                mock.patch.object(self.gc.images,
+                                  'upload') as mock_upload:
+            test_shell.do_image_upload(self.gc, args)
+            mock_upload.assert_called_once_with('IMG-01', 'fileobj',
+                                                1024, backend=None)
+
+    def test_do_image_upload_size_mismatch(self):
+        args = mock.Mock()
+        args.id = 'IMG-01'
+        args.file = 'testfile'
+        args.size = 1024
+        args.store = None
+        args.progress = False
+
+        with mock.patch('glanceclient.common.utils.get_data_file',
+                        return_value='fileobj'), \
+                mock.patch('glanceclient.common.utils.get_file_size',
+                           return_value=2048), \
+                mock.patch('glanceclient.v2.shell._validate_backend'):
+            with self.assertRaisesRegex(ValueError,
+                                        "Size mismatch: provided size 1024 "
+                                        "does not match the size of the "
+                                        "image 2048"):
+                test_shell.do_image_upload(self.gc, args)
+
+    def test_do_image_upload_no_size_in_args(self):
+        args = mock.Mock()
+        args.id = 'IMG-01'
+        args.file = 'testfile'
+        args.size = None
+        args.store = None
+        args.progress = False
+
+        with mock.patch('glanceclient.common.utils.get_data_file',
+                        return_value='fileobj'), \
+                mock.patch('glanceclient.common.utils.get_file_size',
+                           return_value=1024), \
+                mock.patch('glanceclient.v2.shell._validate_backend'), \
+                mock.patch.object(self.gc.images,
+                                  'upload') as mock_upload:
+            test_shell.do_image_upload(self.gc, args)
+            mock_upload.assert_called_once_with('IMG-01', 'fileobj',
+                                                1024, backend=None)
+
     @mock.patch('sys.stdin', autospec=True)
     def test_do_image_create_no_user_props(self, mock_stdin):
         args = self._make_args({'name': 'IMG-01', 'disk_format': 'vhd',
@@ -781,6 +892,77 @@ class ShellV2Test(testtools.TestCase):
             except Exception:
                 pass
 
+    def _do_image_create(self, temp_args, expect_size=None):
+        self.mock_get_data_file.return_value = io.StringIO()
+        with open(tempfile.mktemp(), 'w+') as f:
+            f.write('Some data here')
+            f.flush()
+            f.seek(0)
+            file_name = f.name
+        self.addCleanup(lambda: os.remove(f.name) if os.path.exists(
+            f.name) else None)
+        temp_args = temp_args.copy()
+        temp_args['file'] = file_name
+        args = self._make_args(temp_args)
+        with mock.patch.object(self.gc.images, 'create') as mocked_create, \
+                mock.patch.object(self.gc.images, 'get') as mocked_get, \
+                mock.patch.object(utils, 'get_file_size') as mock_size:
+            expected_size = len('Some data here')
+            mock_size.return_value = expected_size
+            ignore_fields = ['self', 'access', 'schema']
+            expect_image = dict(
+                [(field, field) for field in ignore_fields])
+            expect_image['id'] = 'pass'
+            expect_image['name'] = 'IMG-01'
+            expect_image['disk_format'] = 'vhd'
+            expect_image['container_format'] = 'bare'
+            expect_image['checksum'] = 'fake-checksum'
+            expect_image['os_hash_algo'] = 'fake-hash_algo'
+            expect_image['os_hash_value'] = 'fake-hash_value'
+            if expect_size is not None:
+                expect_image['size'] = expect_size
+            mocked_create.return_value = expect_image
+            mocked_get.return_value = expect_image
+
+            test_shell.do_image_create(self.gc, args)
+
+            temp_args.pop('file', None)
+            mocked_create.assert_called_once_with(**temp_args)
+            mocked_get.assert_called_once_with('pass')
+            expected_dict = {
+                'id': 'pass', 'name': 'IMG-01',
+                'disk_format': 'vhd',
+                'container_format': 'bare',
+                'checksum': 'fake-checksum',
+                'os_hash_algo': 'fake-hash_algo',
+                'os_hash_value': 'fake-hash_value',
+            }
+            if expect_size is not None:
+                expected_dict['size'] = expect_size
+            utils.print_dict.assert_called_once_with(expected_dict)
+            mock_size.assert_called()
+
+    def test_do_image_create_without_size(self):
+        self._do_image_create(
+            temp_args={'name': 'IMG-01', 'disk_format': 'vhd',
+                       'container_format': 'bare', 'progress': False},
+            expect_size=14)
+
+    def test_do_image_create_with_size_exits(self):
+        args = self._make_args({'name': 'test-image', 'size': 1234})
+        expected_msg = ("Setting 'size' during image creation is "
+                        "not supported. Please use --size only when "
+                        "uploading data.")
+        with mock.patch('glanceclient.common.utils.exit') as mock_exit:
+            mock_exit.side_effect = self._mock_utils_exit
+            try:
+                test_shell.do_image_create(self.gc, args)
+                self.fail("utils.exit should have been called")
+            except SystemExit:
+                pass
+
+            mock_exit.assert_called_once_with(expected_msg)
+
     @mock.patch('sys.stdin', autospec=True)
     def test_do_image_create_hidden_image(self, mock_stdin):
         args = self._make_args({'name': 'IMG-01', 'disk_format': 'vhd',
@@ -1652,6 +1834,77 @@ class ShellV2Test(testtools.TestCase):
                         'id': 'via-stdin', 'name': 'Mortimer',
                         'disk_format': 'raw', 'container_format': 'bare'})
 
+    def _image_create_via_import_with_file_helper(
+            self, with_access=True):
+        """Helper for image create via import with file tests."""
+        @mock.patch('glanceclient.common.utils.get_file_size')
+        @mock.patch('glanceclient.v2.shell.do_image_import')
+        @mock.patch('os.access')
+        @mock.patch('sys.stdin', autospec=True)
+        def _test_with_access(mock_stdin, mock_access,
+                              mock_do_import, mock_size):
+            mock_stdin.isatty = lambda: True
+            self.mock_get_data_file.return_value = io.StringIO()
+            mock_access.return_value = with_access
+            mock_size.return_value = 14
+            with open(tempfile.mktemp(), 'w+') as f:
+                f.write('Some data here')
+                f.flush()
+                f.seek(0)
+                file_name = f.name
+            self.addCleanup(
+                lambda: os.remove(file_name) if os.path.exists(
+                    file_name) else None)
+            my_args = self.base_args.copy()
+            my_args.update({'file': file_name})
+            args = self._make_args(my_args)
+            with mock.patch.object(
+                    self.gc.images, 'create') as mocked_create, \
+                    mock.patch.object(
+                        self.gc.images, 'get') as mocked_get, \
+                    mock.patch.object(
+                        self.gc.images, 'get_import_info') as mocked_info:
+                ignore_fields = ['self', 'access', 'schema']
+                expect_image = dict(
+                    [(field, field) for field in ignore_fields])
+                expect_image['id'] = 'fake-image-id'
+                expect_image['name'] = 'Mortimer'
+                expect_image['disk_format'] = 'raw'
+                expect_image['container_format'] = 'bare'
+                mocked_create.return_value = expect_image
+                mocked_get.return_value = expect_image
+                mocked_info.return_value = self.import_info_response
+
+                test_shell.do_image_create_via_import(self.gc, args)
+                mocked_create.assert_called_once()
+                mock_do_import.assert_called_once()
+                mocked_get.assert_called_with(expect_image['id'])
+                mock_size.assert_called_once()
+                utils.print_dict.assert_called_with({
+                    'id': expect_image['id'], 'name': 'Mortimer',
+                    'disk_format': 'raw', 'container_format': 'bare'
+                })
+
+        _test_with_access()
+
+    def test_image_create_via_import_with_file(self):
+        self._image_create_via_import_with_file_helper()
+
+    def test_do_image_create_via_import_with_size_exits(self):
+        args = self._make_args({'name': 'test-image', 'size': 1234})
+        expected_msg = ("Setting 'size' during image creation is not "
+                        "supported. Please use --size only when "
+                        "uploading data.")
+        with mock.patch('glanceclient.common.utils.exit') as mock_exit:
+            mock_exit.side_effect = self._mock_utils_exit
+            try:
+                test_shell.do_image_create(self.gc, args)
+                self.fail("utils.exit should have been called")
+            except SystemExit:
+                pass
+
+            mock_exit.assert_called_once_with(expected_msg)
+
     @mock.patch('glanceclient.v2.shell.do_image_import')
     @mock.patch('glanceclient.v2.shell.do_image_stage')
     @mock.patch('os.access')
@@ -2056,11 +2309,13 @@ class ShellV2Test(testtools.TestCase):
             {'id': 'IMG-01', 'file': 'test', 'size': 1024, 'progress': False})
 
         with mock.patch.object(self.gc.images, 'upload') as mocked_upload:
-            utils.get_data_file = mock.Mock(return_value='testfile')
+            expected_data = '*' * 1024
+            utils.get_data_file = mock.Mock(return_value=expected_data)
+            utils.get_file_size = mock.Mock(return_value=1024)
             mocked_upload.return_value = None
             test_shell.do_image_upload(self.gc, args)
-            mocked_upload.assert_called_once_with('IMG-01', 'testfile', 1024,
+            mocked_upload.assert_called_once_with('IMG-01', expected_data,
-                                                  backend=None)
+                                                  1024, backend=None)
 
     @mock.patch('glanceclient.common.utils.exit')
     def test_image_upload_invalid_store(self, mock_utils_exit):

glanceclient/v2/images.py

@@ -295,12 +295,17 @@ class Controller(object):
 
         :param image_id: ID of the image to upload data for.
         :param image_data: File-like object supplying the data to upload.
-        :param image_size: Unused - present for backwards compatibility
+        :param image_size: If present pass it as header
         :param u_url: Upload url to upload the data to.
         :param backend: Backend store to upload image to.
         """
         url = u_url or '/v2/images/%s/file' % image_id
         hdrs = {'Content-Type': 'application/octet-stream'}
+        if image_size is not None:
+            if not isinstance(image_size, int):
+                raise TypeError("image_size must be an integer, "
+                                "got %s" % type(image_size).__name__)
+            hdrs.update({'x-openstack-image-size': '%i' % image_size})
         if backend is not None:
             hdrs['x-image-meta-store'] = backend
 
@@ -343,11 +348,12 @@ class Controller(object):
 
         :param image_id: ID of the image to upload data for.
         :param image_data: File-like object supplying the data to upload.
-        :param image_size: Unused - present for backwards compatibility
+        :param image_size: If present pass it to upload call
         """
         url = '/v2/images/%s/stage' % image_id
         resp, body = self.upload(image_id,
                                  image_data,
+                                 image_size=image_size,
                                  u_url=url)
         return body, resp
 

glanceclient/v2/shell.py

@@ -49,6 +49,17 @@ def get_image_schema():
     return IMAGE_SCHEMA
 
 
+def get_filesize(args, image_data):
+    filesize = getattr(args, 'size', None) or utils.get_file_size(image_data)
+    if getattr(args, 'size', None) and getattr(args, 'file', None):
+        actual_size = utils.get_file_size(image_data)
+        if actual_size != args.size:
+            raise ValueError("Size mismatch: provided size %s does not match "
+                             "the size of the image %s" % (
+                                 args.size, actual_size))
+    return filesize
+
+
 @utils.schema_args(get_image_schema, omit=['locations', 'os_hidden'])
 # NOTE(rosmaita): to make this option more intuitive for end users, we
 # do not use the Glance image property name 'os_hidden' here.  This means
@@ -66,6 +77,11 @@ def get_image_schema():
            help=_('Local file that contains disk image to be uploaded '
                   'during creation. Alternatively, the image data can be '
                   'passed to the client via stdin.'))
+@utils.arg('--size', metavar='<IMAGE_SIZE>', type=int,
+           help=_('Size in bytes of image to be uploaded. Default is to get '
+                  'size from provided data object but this is supported in '
+                  'case where size cannot be inferred.'),
+           default=None)
 @utils.arg('--progress', action='store_true', default=False,
            help=_('Show upload progress bar.'))
 @utils.arg('--store', metavar='<STORE>',
@@ -90,6 +106,11 @@ def do_image_create(gc, args):
     backend = args.store
 
     file_name = fields.pop('file', None)
+    if 'size' in fields:
+        utils.exit(
+            "Setting 'size' during image creation is not supported. "
+            "Please use --size only when uploading data.")
+
     using_stdin = hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty()
     if args.store and not (file_name or using_stdin):
         utils.exit("--store option should only be provided with --file "
@@ -107,7 +128,6 @@ def do_image_create(gc, args):
         if utils.get_data_file(args) is not None:
             backend = fields.get('store', None)
             args.id = image['id']
-            args.size = None
             do_image_upload(gc, args)
             image = gc.images.get(args.id)
     finally:
@@ -128,6 +148,11 @@ def do_image_create(gc, args):
            help=_('Local file that contains disk image to be uploaded '
                   'during creation. Alternatively, the image data can be '
                   'passed to the client via stdin.'))
+@utils.arg('--size', metavar='<IMAGE_SIZE>', type=int,
+           help=_('Size in bytes of image to be uploaded. Default is to get '
+                  'size from provided data object but this is supported in '
+                  'case where size cannot be inferred.'),
+           default=None)
 @utils.arg('--progress', action='store_true', default=False,
            help=_('Show upload progress bar.'))
 @utils.arg('--import-method', metavar='<METHOD>',
@@ -205,6 +230,11 @@ def do_image_create_via_import(gc, args):
         fields[key] = value
 
     file_name = fields.pop('file', None)
+    if 'size' in fields:
+        utils.exit(
+            "Setting 'size' during image creation is not supported. "
+            "Please use --size only when uploading data.")
+
     using_stdin = hasattr(sys.stdin, 'isatty') and not sys.stdin.isatty()
 
     # special processing for backward compatibility with image-create
@@ -315,7 +345,6 @@ def do_image_create_via_import(gc, args):
         args.id = image['id']
         if args.import_method:
             if utils.get_data_file(args) is not None:
-                args.size = None
                 do_image_stage(gc, args)
             args.from_create = True
             args.stores = stores
@@ -699,13 +728,14 @@ def do_image_upload(gc, args):
         _validate_backend(backend, gc)
 
     image_data = utils.get_data_file(args)
+    filesize = get_filesize(args, image_data)
+
     if args.progress:
-        filesize = utils.get_file_size(image_data)
         if filesize is not None:
             # NOTE(kragniz): do not show a progress bar if the size of the
             # input is unknown (most likely a piped input)
             image_data = progressbar.VerboseFileWrapper(image_data, filesize)
-    gc.images.upload(args.id, image_data, args.size, backend=backend)
+    gc.images.upload(args.id, image_data, filesize, backend=backend)
 
 
 @utils.arg('--file', metavar='<FILE>',
@@ -724,13 +754,14 @@ def do_image_upload(gc, args):
 def do_image_stage(gc, args):
     """Upload data for a specific image to staging."""
     image_data = utils.get_data_file(args)
+    filesize = get_filesize(args, image_data)
+
     if args.progress:
-        filesize = utils.get_file_size(image_data)
         if filesize is not None:
             # NOTE(kragniz): do not show a progress bar if the size of the
             # input is unknown (most likely a piped input)
             image_data = progressbar.VerboseFileWrapper(image_data, filesize)
-    gc.images.stage(args.id, image_data, args.size)
+    gc.images.stage(args.id, image_data, filesize)
 
 
 @utils.arg('--import-method', metavar='<METHOD>', default='glance-direct',