Ensure that every API response item has had the appropriate amount of sleep, even when a generator isn't fully consumed
@@ -230,21 +230,26 @@ class Twarc2:
made_call = time.monotonic()
for response in self.get_paginated(url, params=params):
- # can't return without 'data' if there are no results
- if "data" in response:
- count += len(response["data"])
- yield response
-
- else:
- log.info(f"Retrieved an empty page of results.")
# Calculate the amount of time to sleep, accounting for any
# processing time used by the rest of the application.
# This is to satisfy the 1 request / 1 second rate limit
# on the search/all endpoint.
+ # Note that we're ensuring the appropriate amount of sleep is
+ # taken before yielding every item. This ensures that we won't
+ # exceed the rate limit even in cases where a response generator
+ # is not completely consumed.
time.sleep(max(0, sleep_between - (time.monotonic() - made_call)))
made_call = time.monotonic()
+ # can't return without 'data' if there are no results
+ if "data" in response:
+ count += len(response["data"])
+ yield response
+
+ else:
+ log.info(f"Retrieved an empty page of results.")
+
log.info(f"No more results for search {query}.")
def search_recent(
