Update more tests to use less chunks to speed up all tests which
previously used 1 byte chunk size.
@@ -44,6 +44,19 @@ libcloud.http.DEFAULT_REQUEST_TIMEOUT = 10
MB = 1024 * 1024
+def get_content_iter_with_chunk_size(content, chunk_size=1024):
+ """
+ Return iterator for the provided content which will return chunks of the specified size.
+
+ For performance reasons, larger chunks should be used with very large content to speed up the
+ tests (since in some cases each iteration may result in an TCP send).
+ """
+ # Ensure we still use multiple chunks and iterations
+ assert (len(content) / chunk_size) >= 10
+ content = iter([content[i:i + chunk_size] for i in range(0, len(content), chunk_size)])
+ return content
+
+
class Integration:
class TestBase(unittest.TestCase):
provider = None
@@ -259,11 +272,8 @@ class Integration:
def do_upload(container, blob_name, content):
# NOTE: We originally used a chunk size of 1 which resulted in many requests and as
# such, very slow tests. To speed things up, we use a longer chunk size.
- chunk_size = 1024
- # Ensure we still use multiple chunks and iterations
- assert (len(content) / chunk_size) >= 500
- content = iter([content[i:i + chunk_size] for i in
- range(0, len(content), chunk_size)])
+ assert (len(content) / 1024) >= 500
+ content = get_content_iter_with_chunk_size(content, 1024)
return self.driver.upload_object_via_stream(content, container, blob_name)
def do_download(obj):
@@ -276,7 +286,7 @@ class Integration:
content = gzip.compress(os.urandom(MB // 100))
container = self.driver.create_container(self._random_container_name())
self.driver.upload_object_via_stream(
- iter(content),
+ get_content_iter_with_chunk_size(content, 500),
container,
object_name,
headers={'Content-Encoding': 'gzip'},
@@ -289,7 +299,8 @@ class Integration:
def test_cdn_url(self):
content = os.urandom(MB // 100)
container = self.driver.create_container(self._random_container_name())
- obj = self.driver.upload_object_via_stream(iter(content), container, 'cdn')
+ content_iter = get_content_iter_with_chunk_size(content, 500)
+ obj = self.driver.upload_object_via_stream(content_iter, container, 'cdn')
response = requests.get(self.driver.get_object_cdn_url(obj))
response.raise_for_status()
