[SHARE-495][Fix] PeerJ now respects pages
@@ -3,7 +3,7 @@ import pendulum
import logging
from share import Harvester
-logger = logging.getLogger('com.peerj')
+logger = logging.getLogger(__name__)
class PeerJXMLHarvester(Harvester):
@@ -14,12 +14,16 @@ class PeerJXMLHarvester(Harvester):
while True:
logger.debug('Fetching page %s', url)
resp = self.requests.get(url)
+ resp.raise_for_status()
+ resp_data = resp.json()
- for record in resp.json()['_items']:
+ for record in resp_data['_items']:
if pendulum.parse(record['date']) < start_date:
+ logger.info('%s is before %s, ending harvest', record['date'], start_date)
return
if pendulum.parse(record['date']) > end_date:
+ logger.debug('%s is after %s, skipping', record['date'], end_date)
continue
logger.debug('Fetching article %s', record['_links']['alternate']['xml']['href'])
@@ -28,6 +32,7 @@ class PeerJXMLHarvester(Harvester):
yield record['@id'], details.content
- if 'next' not in record['_links']:
+ if 'next' not in resp_data['_links']:
+ logger.info('No "next" key found, ending harvest')
return
- url = record['_links']['next']['href']
+ url = resp_data['_links']['next']['href']
