Sync versions: create new versions in bulk
Currently we create version per version,
this is calling the db n times (n the number of new versions).
Bulk creation will produce only one call.
To avoid having in memory all versions,
we create them in batch as needed.
@@ -25,7 +25,7 @@ def sync_versions_to_db(project, versions, type): # pylint: disable=redefined-b
- check if user has a ``stable`` / ``latest`` version and disable ours
- update old versions with newer configs (identifier, type, machine)
- - create new versions that do not exist on DB
+ - create new versions that do not exist on DB (in bulk)
- it does not delete versions
:param project: project to update versions
@@ -40,6 +40,7 @@ def sync_versions_to_db(project, versions, type): # pylint: disable=redefined-b
old_versions = dict(old_version_values)
# Add new versions
+ versions_to_create = []
added = set()
has_user_stable = False
has_user_latest = False
@@ -81,7 +82,7 @@ def sync_versions_to_db(project, versions, type): # pylint: disable=redefined-b
identifier=version_id,
type=type,
machine=False,
- ) # noqa
+ )
log.info(
'(Sync Versions) Updated Version: [%s=%s] ',
@@ -90,13 +91,12 @@ def sync_versions_to_db(project, versions, type): # pylint: disable=redefined-b
)
else:
# New Version
- created_version = Version.objects.create(
- project=project,
- type=type,
- identifier=version_id,
- verbose_name=version_name,
+ versions_to_create.append((version_id, version_name))
+
+ added.update(
+ _create_versions_in_bulk(project, type, versions_to_create)
)
- added.add(created_version.slug)
+
if not has_user_stable:
stable_version = (
project.versions.filter(slug=STABLE, type=type).first()
@@ -120,6 +120,32 @@ def sync_versions_to_db(project, versions, type): # pylint: disable=redefined-b
return added
+def _create_versions_in_bulk(project, type, versions):
+ """
+ Create versions (tuple of version_id and version_name) in batch.
+
+ Returns the slug of all added versions.
+ """
+ added = set()
+ batch_size = 150
+ objs = (
+ Version(
+ project=project,
+ type=type,
+ identifier=version_id,
+ verbose_name=version_name,
+ )
+ for version_id, version_name in versions
+ )
+ while True:
+ batch = list(itertools.islice(objs, batch_size))
+ if not batch:
+ break
+ Version.objects.bulk_create(batch, batch_size)
+ added.update(v.slug for v in batch)
+ return added
+
+
def _set_or_create_version(project, slug, version_id, verbose_name, type_):
"""Search or create a version and set its machine attribute to false."""
version = (project.versions.filter(slug=slug).first())
