Merge branch 'release/2.3.0'

Author: chrisseto

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Change Log
 
+## [2.3.0] - 2016-02-02
+## Added
+* Support for OSF Registries
+* New Harvesters
+  * University of Utah
+
+## Changed
+* Updated the API
+* Improved Elasticsearch mappings
+* Updated NIH and NSFAwards
+  * Affiliations are now gathered
+  * Non-Unique URLs are no longer collected
+* Lots of under the hood changes to make dev's lives easier
+
 ## [2.1.0] - 2016-12-16
 ## Added
 * New Harvesters

api/filters.py

@@ -1,7 +1,7 @@
 import django_filters
 import shortuuid
 
-from share.models import ChangeSet, ShareObject, Change
+from share.models import ShareObject
 
 
 class ObjectIDFilter(django_filters.filters.CharFilter):
@@ -18,27 +18,3 @@ class ShareObjectFilterSet(django_filters.FilterSet):
     class Meta:
         model = ShareObject
         fields = ['object_id', ]
-
-
-class ChangeSetFilterSet(django_filters.FilterSet):
-    status = django_filters.MethodFilter()
-    target_uuid = django_filters.filters.UUIDFilter(name='changes__share_objects__uuid')
-    submitted_by = django_filters.filters.NumberFilter(name='normalized_data__source')
-
-    def filter_status(self, queryset, value):
-        # django-filters ChoicesFilter doesn't actually work, at least not with django-model-utils choices
-        if value and hasattr(ChangeSet.STATUS, value):
-            return queryset.filter(status=getattr(ChangeSet.STATUS, value))
-        return queryset
-
-    class Meta:
-        model = ChangeSet
-        fields = ['submitted_by', 'status', 'target_uuid']
-
-
-class ChangeFilterSet(django_filters.FilterSet):
-    changeset = django_filters.filters.NumberFilter(name='change_set_id')
-
-    class Meta:
-        model = Change
-        fields = ['changeset']

api/renderers.py

@@ -3,20 +3,24 @@
 from collections import OrderedDict
 
 from django.utils import encoding
+from django.apps import apps
 
 # from rest_framework.compat import SHORT_SEPARATORS, LONG_SEPARATORS, INDENT_SEPARATORS
 from rest_framework.renderers import JSONRenderer
 from rest_framework.settings import api_settings
 from rest_framework.utils import encoders
 from rest_framework import relations
-from rest_framework.serializers import BaseSerializer
+from rest_framework.serializers import BaseSerializer, Serializer, ListSerializer
 
 from rest_framework_json_api.renderers import JSONRenderer as JSONAPIRenderer
 from rest_framework_json_api import utils
 
+from share.util import IDObfuscator
+
 
 class HideNullJSONAPIRenderer(JSONAPIRenderer):
 
+    # override null behavior from JSONAPIRenderer
     @staticmethod
     def extract_attributes(fields, resource):
         data = OrderedDict()
@@ -46,11 +50,201 @@ def extract_attributes(fields, resource):
 
         return utils.format_keys(data)
 
+    def encode_id(resource_id, resource_type):
+        return encoding.force_text(IDObfuscator.encode_id(resource_id, apps.get_model('share', resource_type)))
+
+    @classmethod
+    def encode_ids(cls, relation_data):
+        if relation_data:
+            if isinstance(relation_data, list):
+                for obj in relation_data:
+                    obj['id'] = cls.encode_id(int(obj['id']), obj['type'])
+            else:
+                relation_data['id'] = cls.encode_id(int(relation_data['id']), relation_data['type'])
+        return relation_data
+
+    # override ids in relationships from JSONAPIRenderer
+    @classmethod
+    def extract_relationships(cls, fields, resource, resource_instance):
+        # Avoid circular deps
+        from rest_framework_json_api.relations import ResourceRelatedField
+
+        data = OrderedDict()
+
+        # Don't try to extract relationships from a non-existent resource
+        if resource_instance is None:
+            return
+
+        for field_name, field in six.iteritems(fields):
+            # Skip URL field
+            if field_name == api_settings.URL_FIELD_NAME:
+                continue
+
+            # Skip fields without relations
+            if not isinstance(field, (relations.RelatedField, relations.ManyRelatedField, BaseSerializer)):
+                continue
+
+            source = field.source
+            relation_type = utils.get_related_resource_type(field)
+
+            if isinstance(field, relations.HyperlinkedIdentityField):
+                resolved, relation_instance = utils.get_relation_instance(resource_instance, source, field.parent)
+                if not resolved:
+                    continue
+                # special case for HyperlinkedIdentityField
+                relation_data = list()
+
+                # Don't try to query an empty relation
+                relation_queryset = relation_instance \
+                    if relation_instance is not None else list()
+
+                for related_object in relation_queryset:
+                    relation_data.append(
+                        OrderedDict([('type', relation_type), ('id', encoding.force_text(related_object.pk))])
+                    )
+
+                data.update({field_name: {
+                    'links': {
+                        'related': resource.get(field_name)},
+                    'data': cls.encode_ids(relation_data),
+                    'meta': {
+                        'count': len(relation_data)
+                    }
+                }})
+                continue
+
+            if isinstance(field, ResourceRelatedField):
+                resolved, relation_instance = utils.get_relation_instance(resource_instance, source, field.parent)
+                if not resolved:
+                    continue
+
+                # special case for ResourceRelatedField
+                relation_data = {
+                    'data': cls.encode_ids(resource.get(field_name))
+                }
+
+                field_links = field.get_links(resource_instance)
+                relation_data.update(
+                    {'links': field_links}
+                    if field_links else dict()
+                )
+                data.update({field_name: relation_data})
+                continue
+
+            if isinstance(field, (relations.PrimaryKeyRelatedField, relations.HyperlinkedRelatedField)):
+                resolved, relation = utils.get_relation_instance(resource_instance, '%s_id' % source, field.parent)
+                if not resolved:
+                    continue
+                relation_id = relation if resource.get(field_name) else None
+                relation_data = {
+                    'data': (
+                        OrderedDict([('type', relation_type), ('id', cls.encode_id(relation_id, relation_type))])
+                        if relation_id is not None else None)
+                }
+
+                relation_data.update(
+                    {'links': {'related': resource.get(field_name)}}
+                    if isinstance(field, relations.HyperlinkedRelatedField) and resource.get(field_name) else dict()
+                )
+                data.update({field_name: relation_data})
+                continue
+
+            if isinstance(field, relations.ManyRelatedField):
+                resolved, relation_instance = utils.get_relation_instance(resource_instance, source, field.parent)
+                if not resolved:
+                    continue
+
+                if isinstance(field.child_relation, ResourceRelatedField):
+                    # special case for ResourceRelatedField
+                    relation_data = {
+                        'data': cls.encode_ids(resource.get(field_name))
+                    }
+
+                    field_links = field.child_relation.get_links(resource_instance)
+                    relation_data.update(
+                        {'links': field_links}
+                        if field_links else dict()
+                    )
+                    relation_data.update(
+                        {
+                            'meta': {
+                                'count': len(resource.get(field_name))
+                            }
+                        }
+                    )
+                    data.update({field_name: relation_data})
+                    continue
+
+                relation_data = list()
+                for nested_resource_instance in relation_instance:
+                    nested_resource_instance_type = (
+                        relation_type or
+                        utils.get_resource_type_from_instance(nested_resource_instance)
+                    )
+
+                    relation_data.append(OrderedDict([
+                        ('type', nested_resource_instance_type),
+                        ('id', encoding.force_text(nested_resource_instance.pk))
+                    ]))
+                data.update({
+                    field_name: {
+                        'data': cls.encode_ids(relation_data),
+                        'meta': {
+                            'count': len(relation_data)
+                        }
+                    }
+                })
+                continue
+
+            if isinstance(field, ListSerializer):
+                resolved, relation_instance = utils.get_relation_instance(resource_instance, source, field.parent)
+                if not resolved:
+                    continue
+
+                relation_data = list()
+
+                serializer_data = resource.get(field_name)
+                resource_instance_queryset = list(relation_instance)
+                if isinstance(serializer_data, list):
+                    for position in range(len(serializer_data)):
+                        nested_resource_instance = resource_instance_queryset[position]
+                        nested_resource_instance_type = (
+                            relation_type or
+                            utils.get_resource_type_from_instance(nested_resource_instance)
+                        )
+
+                        relation_data.append(OrderedDict([
+                            ('type', nested_resource_instance_type),
+                            ('id', encoding.force_text(nested_resource_instance.pk))
+                        ]))
+
+                    data.update({field_name: {'data': cls.encode_ids(relation_data)}})
+                    continue
+
+            if isinstance(field, Serializer):
+                resolved, relation_instance = utils.get_relation_instance(resource_instance, source, field.parent)
+                if not resolved:
+                    continue
+
+                data.update({
+                    field_name: {
+                        'data': (
+                            OrderedDict([
+                                ('type', relation_type),
+                                ('id', cls.encode_id(resource_instance.pk, relation_type))
+                            ]) if resource.get(field_name) else None)
+                    }
+                })
+                continue
+
+        return utils.format_keys(data)
+
+    # override top level id from JSONAPIRenderer
     @classmethod
     def build_json_resource_obj(cls, fields, resource, resource_instance, resource_name):
         resource_data = [
             ('type', resource_name),
-            ('id', encoding.force_text(resource_instance.pk) if resource_instance else None),
+            ('id', cls.encode_id(resource_instance.pk, resource_instance._meta.model.__name__) if resource_instance and resource_instance.pk else None),
             ('attributes', cls.extract_attributes(fields, resource)),
         ]
         relationships = cls.extract_relationships(fields, resource, resource_instance)

api/serializers.py

@@ -5,7 +5,55 @@
 from rest_framework_json_api import serializers
 
 from share import models
-from share.models import ChangeSet, ProviderRegistration, CeleryProviderTask, SiteBanner
+from share.models import ProviderRegistration, CeleryProviderTask, SiteBanner
+
+from api import fields
+
+
+class BaseShareSerializer(serializers.ModelSerializer):
+
+    def __init__(self, *args, **kwargs):
+        # super hates my additional kwargs
+        sparse = kwargs.pop('sparse', False)
+        version_serializer = kwargs.pop('version_serializer', False)
+        super(BaseShareSerializer, self).__init__(*args, **kwargs)
+
+        if sparse:
+            # clear the fields if they asked for sparse
+            self.fields.clear()
+        else:
+            # remove hidden fields
+            excluded_fields = ['change', 'sources']
+            for field_name in tuple(self.fields.keys()):
+                if 'version' in field_name or field_name in excluded_fields:
+                    self.fields.pop(field_name)
+
+            if not version_serializer:
+                # add links to related objects
+                self.fields.update({
+                    'links': fields.LinksField(links=self.Meta.links, source='*')
+                })
+
+        # version specific fields
+        if version_serializer:
+            self.fields.update({
+                'action': serializers.CharField(max_length=10),
+                'persistent_id': serializers.IntegerField()
+            })
+
+        # add fields with improper names
+        self.fields.update({
+            'type': fields.TypeField(),
+        })
+
+    class Meta:
+        links = ('versions', 'changes', 'rawdata')
+
+    # http://stackoverflow.com/questions/27015931/remove-null-fields-from-django-rest-framework-response
+    def to_representation(self, instance):
+        ret = super(BaseShareSerializer, self).to_representation(instance)
+        ret = OrderedDict(list(filter(lambda x: x[1] is not None, ret.items())))
+        return ret
 
 
 class ShareModelSerializer(serializers.ModelSerializer):
@@ -57,15 +105,6 @@ class Meta:
         fields = ('data', 'source')
 
 
-class ChangeSerializer(ShareModelSerializer):
-    self = serializers.HyperlinkedIdentityField(view_name='api:change-detail')
-    target_type = serializers.StringRelatedField()
-
-    class Meta:
-        model = models.Change
-        fields = ('self', 'id', 'change', 'node_id', 'type', 'target_type', 'target_id')
-
-
 class ShareUserSerializer(ShareModelSerializer):
     def __init__(self, *args, token=None, **kwargs):
         super(ShareUserSerializer, self).__init__(*args, **kwargs)
@@ -100,24 +139,6 @@ class Meta:
         )
 
 
-class ChangeSetSerializer(ShareModelSerializer):
-    # changes = ChangeSerializer(many=True)
-    change_count = serializers.SerializerMethodField()
-    self = serializers.HyperlinkedIdentityField(view_name='api:changeset-detail')
-    source = ShareUserSerializer(source='normalized_data.source')
-    status = serializers.SerializerMethodField()
-
-    def get_status(self, obj):
-        return ChangeSet.STATUS[obj.status]
-
-    def get_change_count(self, obj):
-        return obj.changes.count()
-
-    class Meta:
-        model = models.ChangeSet
-        fields = ('self', 'id', 'submitted_at', 'change_count', 'source', 'status')
-
-
 class ProviderSerializer(ShareUserSerializer):
     def __init__(self, *args, **kwargs):
         super(ShareUserSerializer, self).__init__(*args, **kwargs)