use full request url in metadata in paged responses
@@ -429,12 +429,13 @@ class Twarc2:
Yields one page (one API response) at a time.
"""
- page = self.get(*args, **kwargs).json()
+ resp = self.get(*args, **kwargs)
+ page = resp.json()
url = args[0]
if self.metadata:
- page = _append_metadata(page, page.url)
+ page = _append_metadata(page, resp.url)
yield page
@@ -454,10 +455,11 @@ class Twarc2:
else:
kwargs["params"] = {token_param: page["meta"]["next_token"]}
- page = self.get(*args, **kwargs).json()
+ resp = self.get(*args, **kwargs)
+ page = resp.json()
if self.metadata:
- page = _append_metadata(page, page.url)
+ page = _append_metadata(page, resp.url)
yield page
