Merge branch 'hotfix/2.4.2'

Author: chrisseto

providers/com/peerj/xml/harvester.py

@@ -3,7 +3,7 @@
 import logging
 from share import Harvester
 
-logger = logging.getLogger('com.peerj')
+logger = logging.getLogger(__name__)
 
 
 class PeerJXMLHarvester(Harvester):
@@ -14,12 +14,16 @@ def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum)
         while True:
             logger.debug('Fetching page %s', url)
             resp = self.requests.get(url)
+            resp.raise_for_status()
+            resp_data = resp.json()
 
-            for record in resp.json()['_items']:
+            for record in resp_data['_items']:
                 if pendulum.parse(record['date']) < start_date:
+                    logger.info('%s is before %s, ending harvest', record['date'], start_date)
                     return
 
                 if pendulum.parse(record['date']) > end_date:
+                    logger.debug('%s is after %s, skipping', record['date'], end_date)
                     continue
 
                 logger.debug('Fetching article %s', record['_links']['alternate']['xml']['href'])
@@ -28,6 +32,7 @@ def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum)
 
                 yield record['@id'], details.content
 
-            if 'next' not in record['_links']:
+            if 'next' not in resp_data['_links']:
+                logger.info('No "next" key found, ending harvest')
                 return
-            url = record['_links']['next']['href']
+            url = resp_data['_links']['next']['href']