document reason for backoff in retry
@@ -5,8 +5,36 @@ import rdflib
import rdflib.query
-MAX_RETRY = 3
+MAX_RETRY = 10
+BACKOFF_FACTOR = 1.5
def query_with_retry(graph: rdflib.Graph, query: str, **kwargs) -> rdflib.query.Result:
+ """Query graph an retry on failure, returns preloaded result
+
+ The tests run against outside network targets which results
+ in flaky tests. Therefor retries are needed to increase stability.
+
+ There are two main categories why these might fail:
+
+ * Resource shortage on the server running the tests (e.g. out of ports)
+ * Issues outside the server (network, target server, etc)
+
+ As fast feedback is important the retry should be done quickly.
+ Therefor the first retry is done after 100ms. But if the issue is
+ outside the server running the tests it we need to be good
+ citizenship of the internet and not hit servers of others at
+ a constant rate. (Also it might get us banned)
+
+ Therefor this function implements a backoff mechanism.
+
+ When adjusting the parameters please keep in mind that several
+ tests might run on the same machine at the same time
+ on our CI, and we really don't want to hit any rate limiting.
+
+ The maximum time the function waits is:
+
+ >>> sum((BACKOFF_FACTOR ** backoff) / 10 for backoff in range(MAX_RETRY))
+ 11.3330078125
+ """
backoff = 0
for i in range(MAX_RETRY):
try:
@@ -17,7 +45,7 @@ def query_with_retry(graph: rdflib.Graph, query: str, **kwargs) -> rdflib.query.
if i == MAX_RETRY -1:
raise e
- backoff_s = 1.2 ** backoff
- print(f"Network eroror {e} during query, waiting for {backoff_s}s and retrying")
- time.sleep(1)
+ backoff_s = (BACKOFF_FACTOR ** backoff) / 10
+ print(f"Network error {e} during query, waiting for {backoff_s:.2f}s and retrying")
+ time.sleep(backoff_s)
backoff += 1
