Merge pull request #11654 from CenterForOpenScience/feature/post-nr-post-release

[ENG-9856][ENG-10620][ENG-10654] Post-postNR hotfixes

Author: cslzchen

.gitignore

@@ -122,6 +122,7 @@ website/static/sitemaps/
 ##############
 
 api/static/vendor
+osf/features.override.yaml
 
 # Local settings files
 local.py

README-docker-compose.md

@@ -294,6 +294,11 @@
       ```bash
       docker compose run --rm web python3 manage.py reset_db --noinput
       ```
+- OPTIONAL: Create a `features.yaml` override
+  ```bash
+  cp osf/features.yaml osf/features.override.yaml
+  ```
+  Note: Update the WAFFLE_VALUES_YAML variable to point to the override file (osf/features.override.yaml).
 
 ## Application Debugging
 

notifications/tasks.py

@@ -497,18 +497,42 @@ def send_no_addon_email(self, dry_run=False, **kwargs):
 
 @celery_app.task(bind=True, name='notifications.tasks.notifications_cleanup_task')
 def notifications_cleanup_task(self, dry_run=False, **kwargs):
-    """Remove old notifications and email tasks from the database."""
-
+    """Entry point for notifications and email tasks cleanup task."""
     cutoff_date = timezone.now() - settings.NOTIFICATIONS_CLEANUP_AGE
-    old_notifications = Notification.objects.filter(sent__lt=cutoff_date)
-    old_email_tasks = EmailTask.objects.filter(created_at__lt=cutoff_date)
-
     if dry_run:
-        notifications_count = old_notifications.count()
-        email_tasks_count = old_email_tasks.count()
+        notifications_count = Notification.objects.filter(sent__lt=cutoff_date).count()
+        email_tasks_count = EmailTask.objects.filter(created_at__lt=cutoff_date).count()
         logger.info(f'[Dry Run] Notifications Cleanup Task: {notifications_count} notifications and {email_tasks_count} email tasks would be deleted.')
         return
 
-    deleted_notifications_count, _ = old_notifications.delete()
-    deleted_email_tasks_count, _ = old_email_tasks.delete()
-    logger.info(f'Notifications Cleanup Task: Deleted {deleted_notifications_count} notifications and {deleted_email_tasks_count} email tasks older than {cutoff_date}.')
+    delete_batch.delay('osf', 'Notification', {'sent__lt': cutoff_date}, batch_size=settings.NOTIFICATIONS_CLEANUP_BATCH_SIZE)
+    delete_batch.delay('osf', 'EmailTask', {'created_at__lt': cutoff_date}, batch_size=settings.NOTIFICATIONS_CLEANUP_BATCH_SIZE)
+
+
+@celery_app.task(bind=True, name='notifications.tasks.delete_batch')
+def delete_batch(
+    self,
+    app_label,
+    model_name,
+    filters,
+    order_field='id',
+    batch_size=10000
+):
+    Model = apps.get_model(app_label, model_name)
+
+    ids = list(
+        Model.objects
+        .filter(**filters)
+        .order_by(order_field)
+        .values_list('id', flat=True)[:batch_size]
+    )
+
+    if not ids:
+        logger.info(f'{model_name} cleanup finished')
+        return
+
+    deleted, _ = Model.objects.filter(id__in=ids).delete()
+
+    logger.info(f'Deleted {deleted} rows from {model_name}')
+
+    delete_batch.delay(app_label, model_name, filters, order_field, batch_size)

osf/management/commands/remove_duplicate_notification_subscriptions.py

@@ -7,6 +7,11 @@
 from django.contrib.contenttypes.models import ContentType
 from framework.celery_tasks import app as celery_app
 from osf.models import NotificationSubscription
+from django.db.models import Exists, OuterRef
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
 
 
 @celery_app.task(name='scripts.remove_after_use.merge_notification_subscription_provider_ct')
@@ -19,13 +24,37 @@ def merge_notification_subscription_provider_ct():
         ContentType.objects.get_by_natural_key('osf', 'registrationprovider'),
         ContentType.objects.get_by_natural_key('osf', 'collectionprovider'),
     ]
-    subscriptions = NotificationSubscription.objects.filter(
-        content_type__in=provider_ct_list
-    )
-    subscriptions.update(
+
+    provider_ct_ids = [ct.id for ct in provider_ct_list]
+
+    abstract_provider_ct_qs = NotificationSubscription.objects.filter(
         content_type=abstract_provider_ct
     )
 
+    duplicates = NotificationSubscription.objects.filter(
+        content_type_id__in=provider_ct_ids
+    ).annotate(
+        abstract_exists=Exists(
+            abstract_provider_ct_qs.filter(
+                notification_type_id=OuterRef('notification_type_id'),
+                user_id=OuterRef('user_id'),
+                object_id=OuterRef('object_id'),
+                _is_digest=OuterRef('_is_digest'),
+            )
+        )
+    ).filter(abstract_exists=True)
+
+    # delete rows that would conflict
+    logger.info(f'Deleted {duplicates.count()} duplicate NotificationSubscription rows with provider content types.')
+    duplicates.delete()
+
+    # update remaining rows
+    update_qs = NotificationSubscription.objects.filter(
+        content_type_id__in=provider_ct_ids
+    )
+    logger.info(f'Updated {update_qs.count()} NotificationSubscription rows to use abstract provider content type.')
+    update_qs.update(content_type=abstract_provider_ct)
+
 
 if __name__ == '__main__':
     merge_notification_subscription_provider_ct.delay()