Improve error handling during imports
Capture errors and keep trucking
log errors with actual trigger data and full traceback
@@ -26,12 +26,10 @@ import datetime
import logging
from collections import Counter
from itertools import chain
-from typing import Dict
-from typing import List
+import traceback
from typing import Set
from typing import Tuple
from typing import Optional
-from typing import Sequence
import packageurl
from django.db import DataError
@@ -144,6 +142,7 @@ def process_advisories(data_source: DataSource) -> None:
advisory_batches = chain(data_source.updated_advisories(), data_source.added_advisories())
for batch in advisory_batches:
for advisory in batch:
+ try:
vuln, vuln_created = _get_or_create_vulnerability(advisory)
for vuln_ref in advisory.vuln_references:
ref = VulnerabilityReferenceInserter(
@@ -180,6 +179,9 @@ def process_advisories(data_source: DataSource) -> None:
if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
handle_conflicts([existing_ref[0], pkg_vuln_ref.to_model_object()])
existing_ref.delete()
+ except Exception as e:
+ # TODO: store error but continue
+ logger.error(f"Failed to process advisory: {advisory}:\n" + traceback.format_exc())
models.VulnerabilityReference.objects.bulk_create(
[i.to_model_object() for i in bulk_create_vuln_refs]
@@ -242,6 +244,7 @@ def handle_conflicts(conflicts):
def _get_or_create_vulnerability(
advisory: Advisory,
) -> Tuple[models.Vulnerability, bool]:
+
if advisory.cve_id:
query_kwargs = {"cve_id": advisory.cve_id}
elif advisory.summary:
@@ -249,15 +252,18 @@ def _get_or_create_vulnerability(
else:
return models.Vulnerability.objects.create(), True
+ try:
vuln, created = models.Vulnerability.objects.get_or_create(**query_kwargs)
-
# Eventually we only want to keep summary from NVD and ignore other descriptions.
if advisory.summary and vuln.summary != advisory.summary:
vuln.summary = advisory.summary
vuln.save()
-
return vuln, created
+ except Exception as e:
+ logger.error(f"Failed to _get_or_create_vulnerability: {query_kwargs!r}:\n" + traceback.format_exc())
+ raise
+
def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:
version = p.version
