[SHARE-529][SHARE-590][Feature] Use cursor, not offset, in Crossref harvester. (#581)

* Use cursor, not offset, in crossref harvester.
* Don't hold all harvested data in memory.

Author: aaxelb

providers/org/crossref/harvester.py

@@ -15,26 +15,21 @@ def do_harvest(self, start_date, end_date):
                 start_date.isoformat(),
                 end_date.isoformat()
             ),
-            'rows': 1000
-        }).url)
-
-    def fetch_records(self, url):
-        resp = self.requests.get(url)
-        resp.raise_for_status()
-        total = resp.json()['message']['total-results']
-        records = resp.json()['message']['items']
-
-        # return the first 1000 records
-        for record in records:
-            yield (record['DOI'], record)
-
-        # make requests for the remaining records
-        for i in range(1000, total, 1000):
-            response = self.requests.get(furl(url).add(query_params={
-                'offset': i
-            }).url)
-
-            response.raise_for_status()
-            records = response.json()['message']['items']
+            'rows': 1000,
+        }))
+
+    def fetch_records(self, url: furl):
+        cursor = '*'
+
+        while True:
+            url.args['cursor'] = cursor
+            resp = self.requests.get(url.url)
+            resp.raise_for_status()
+            message = resp.json()['message']
+            records = message['items']
+            cursor = message['next-cursor']
+
+            if not records:
+                break
             for record in records:
                 yield (record['DOI'], record)

share/harvest/harvester.py

@@ -122,17 +122,17 @@ def harvest(self, start_date: [datetime.datetime, datetime.timedelta, pendulum.P
         from share.models import RawData
         start_date, end_date = self._validate_dates(start_date, end_date)
 
-        stored = []
+        raw_ids = []
         with transaction.atomic():
             rawdata = self.do_harvest(start_date, end_date, **kwargs)
             assert isinstance(rawdata, types.GeneratorType), 'do_harvest did not return a generator type, found {!r}. Make sure to use the yield keyword'.format(type(rawdata))
 
             for doc_id, datum in rawdata:
-                stored.append(RawData.objects.store_data(doc_id, self.encode_data(datum), self.source, self.config.label))
-                if limit is not None and len(stored) >= limit:
+                raw_ids.append(RawData.objects.store_data(doc_id, self.encode_data(datum), self.source, self.config.label).id)
+                if limit is not None and len(raw_ids) >= limit:
                     break
 
-        return stored
+        return raw_ids
 
     def raw(self, start_date: [datetime.datetime, datetime.timedelta, pendulum.Pendulum], end_date: [datetime.datetime, datetime.timedelta, pendulum.Pendulum], shift_range: bool=True, limit: int=None, **kwargs) -> list:
         start_date, end_date = self._validate_dates(start_date, end_date)

share/tasks.py

@@ -137,18 +137,20 @@ def do_run(self, start: [str, datetime.datetime]=None, end: [str, datetime.datet
 
         try:
             logger.info('Starting harvester run for %s %s - %s', self.config.label, start, end)
-            raws = harvester.harvest(start, end, limit=limit, **kwargs)
-            logger.info('Collected %d data blobs from %s', len(raws), self.config.label)
+            raw_ids = harvester.harvest(start, end, limit=limit, **kwargs)
+            logger.info('Collected %d data blobs from %s', len(raw_ids), self.config.label)
         except Exception as e:
             logger.exception('Failed harvester task (%s, %s, %s)', self.config.label, start, end)
             raise self.retry(countdown=10, exc=e)
 
-        for raw in raws:
-            # attach task
-            raw.tasks.add(self.task)
-
-            task = NormalizerTask().apply_async((self.started_by.id, self.config.label, raw.pk,))
-            logger.debug('Started normalizer task %s for %s', task, raw.id)
+        # attach task to each RawData
+        RawData.tasks.through.objects.bulk_create([
+            RawData.tasks.through(rawdata_id=raw_id, celeryprovidertask_id=self.task.id)
+            for raw_id in raw_ids
+        ])
+        for raw_id in raw_ids:
+            task = NormalizerTask().apply_async((self.started_by.id, self.config.label, raw_id,))
+            logger.debug('Started normalizer task %s for %s', task, raw_id)
 
 
 class NormalizerTask(AppTask):