Production release: lab page info, jsPsych survey likert, package updates

.gitignore

@@ -154,3 +154,8 @@ data
 
 bootstrap-5*
 .python-version
+
+.aws/*
+.claude/
+scratch/*
+aws-lambda/*

accounts/models.py

@@ -1,9 +1,12 @@
 import base64
 import hashlib
+import logging
 import uuid
 from io import BytesIO
 from typing import Union
 
+logger = logging.getLogger(__name__)
+
 import pydenticon
 import pyotp
 from bitfield import BitField
@@ -681,6 +684,12 @@ def send_as_email(self):
         recipient_email_list = list(self.recipients.values_list("username", flat=True))
 
         for to_email in recipient_email_list:
+            if not to_email or "@" not in to_email:
+                logger.warning(
+                    "Skipping custom email recipient: username %s is not a valid email address.",
+                    to_email,
+                )
+                continue
             user = User.objects.get(username=to_email)
             context.update(token=user.generate_token(), username=to_email)
             send_mail.delay(

exp/views/lab.py

@@ -73,6 +73,7 @@ def get_context_data(self, **kwargs):
         context["custom_url"] = self.request.build_absolute_uri(
             reverse("web:lab-studies-list", args=[lab.slug])
         )
+        context["lab_stats"] = lab.get_response_stats()
         return context
 
 

pyproject.toml

@@ -7,7 +7,7 @@ requires-python = "==3.13.*"
 dependencies = [
     "boto3==1.40.52",
     "ciso8601==2.3.2",
-    "django==5.2.13",
+    "django==5.2.14",
     "django-ace-overlay",
     "django-bitfield==2.2.0",
     "django-bootstrap-icons==0.9.0",

studies/forms.py

@@ -120,6 +120,15 @@ class Meta:
             "badge": "This image will be shown at the top of your custom URL page when it is viewed on a mobile device/narrow browser window, and as a badge/avatar image for your lab. This image should be square. Please keep your file size less than 1 MB.",
         }
 
+    def clean_description(self):
+        """Lazy, on-demand fix for malformed lab description values in the database that contain literal \r\n strings."""
+        description = self.cleaned_data.get("description", "")
+        return (
+            description.replace("\\r\\n", "\n")
+            .replace("\\r", "\n")
+            .replace("\\n", "\n")
+        )
+
     def clean_banner(self):
         cleaned_banner = self.cleaned_data["banner"]
         ratio = 2

studies/management/commands/delete_lab.py

@@ -0,0 +1,153 @@
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from guardian.models import GroupObjectPermission, UserObjectPermission
+
+from studies.models import Lab
+
+EXPECTED_RELS = {
+    ("Study", "lab"),
+    ("LabUserObjectPermission", "content_object"),
+    ("LabGroupObjectPermission", "content_object"),
+}
+
+
+class Command(BaseCommand):
+    help = "Safely delete a Lab and its associated permissions and groups."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--uuid", type=str, required=True, help="UUID of the Lab to delete."
+        )
+        parser.add_argument(
+            "--keep-groups",
+            action="store_true",
+            help="Preserve the Lab's auth Groups after deletion.",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Preview deletion without deleting.",
+        )
+        parser.add_argument(
+            "--force",
+            action="store_true",
+            help="Actually perform deletion.",
+        )
+
+    def handle(self, *args, **options):
+        uuid = options["uuid"]
+        keep_groups = options["keep_groups"]
+        dry_run = options["dry_run"]
+        force = options["force"]
+
+        if not dry_run and not force:
+            raise CommandError(
+                "Refusing to delete without --force. Use --dry-run to preview."
+            )
+
+        self.stdout.write("Inspecting relations pointing to Lab...\n")
+        found_rels = set()
+        for rel in Lab._meta.related_objects:
+            model_name = rel.related_model.__name__
+            field_name = rel.field.name
+            on_delete = rel.field.remote_field.on_delete.__name__
+            nullable = rel.field.null
+            found_rels.add((model_name, field_name))
+            self.stdout.write(
+                f"- {model_name}.{field_name} (on_delete={on_delete}, nullable={nullable})"
+            )
+
+        unexpected = found_rels - EXPECTED_RELS
+        if unexpected:
+            raise CommandError(
+                f"Refusing to delete Lab. Unexpected relations detected: {unexpected}"
+            )
+
+        try:
+            lab = Lab.objects.get(uuid=uuid)
+        except (Lab.DoesNotExist, ValidationError):
+            raise CommandError(f"Lab with UUID {uuid} is invalid or not found.")
+
+        self.stdout.write(self.style.WARNING(f"\nLab: {lab.name} ({lab.uuid})"))
+
+        study_count = lab.studies.count()
+        if study_count > 0:
+            raise CommandError(
+                f"Refusing to delete lab: {study_count} studies are still attached."
+            )
+
+        lab_ct = ContentType.objects.get_for_model(Lab)
+        user_perm_count = UserObjectPermission.objects.filter(
+            content_type=lab_ct, object_pk=str(lab.pk)
+        ).count()
+        group_perm_count = GroupObjectPermission.objects.filter(
+            content_type=lab_ct, object_pk=str(lab.pk)
+        ).count()
+        researcher_count = lab.researchers.count()
+        requested_count = lab.requested_researchers.count()
+        groups = [
+            g
+            for g in [
+                lab.guest_group,
+                lab.readonly_group,
+                lab.member_group,
+                lab.admin_group,
+            ]
+            if g is not None
+        ]
+
+        self.stdout.write(f"  User object permissions to delete: {user_perm_count}")
+        self.stdout.write(f"  Group object permissions to delete: {group_perm_count}")
+        self.stdout.write(f"  Researchers to remove: {researcher_count}")
+        self.stdout.write(f"  Pending join requests to remove: {requested_count}")
+        self.stdout.write(
+            f"  Auth groups to {'preserve' if keep_groups else 'delete'}: {len(groups)}"
+        )
+        self.stdout.write(f"  Badge image to delete: {lab.badge.name or 'none'}")
+        self.stdout.write(f"  Banner image to delete: {lab.banner.name or 'none'}")
+
+        if dry_run:
+            self.stdout.write(
+                self.style.SUCCESS("\nDry run complete. No deletion performed.")
+            )
+            return
+
+        with transaction.atomic():
+            lab = Lab.objects.select_for_update().get(uuid=uuid)
+
+            UserObjectPermission.objects.filter(
+                content_type=lab_ct, object_pk=str(lab.pk)
+            ).delete()
+            GroupObjectPermission.objects.filter(
+                content_type=lab_ct, object_pk=str(lab.pk)
+            ).delete()
+
+            lab.researchers.clear()
+            lab.requested_researchers.clear()
+
+            if lab.badge:
+                lab.badge.delete(save=False)
+            if lab.banner:
+                lab.banner.delete(save=False)
+
+            groups = [
+                g
+                for g in [
+                    lab.guest_group,
+                    lab.readonly_group,
+                    lab.member_group,
+                    lab.admin_group,
+                ]
+                if g is not None
+            ]
+            lab.delete()
+            self.stdout.write("Lab deleted.")
+
+            if not keep_groups:
+                for group in groups:
+                    self.stdout.write(f"Deleting group: {group.name}")
+                    group.delete()
+
+        self.stdout.write(self.style.SUCCESS("Done."))

studies/migrations/0107_add_lab_created_at.py

@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("studies", "0106_add_researcher_valid_response"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="lab",
+            name="created_at",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

studies/models.py

@@ -1,6 +1,6 @@
 import logging
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from enum import Enum
 
 import boto3
@@ -54,6 +54,12 @@
 REJECTED = "rejected"
 CONSENT_RULINGS = (ACCEPTED, REJECTED, PENDING)
 
+# It would've made more sense to put this in queries.py and import it here, but queries.py imports from this file, so doing it the other way around avoids a circular import.
+EFFECTIVELY_TALLIED_Q = models.Q(
+    is_tallied_researcher_override__isnull=False,
+    is_tallied_researcher_override=True,
+) | models.Q(is_tallied_researcher_override__isnull=True, is_tallied=True)
+
 S3_RESOURCE = boto3.resource("s3")
 S3_BUCKET = S3_RESOURCE.Bucket(settings.BUCKET_NAME)
 
@@ -155,6 +161,7 @@ class Lab(models.Model):
     )
     banner = models.ImageField(null=True, blank=True, upload_to="lab_images/")
     badge = models.ImageField(null=True, blank=True, upload_to="lab_images/")
+    created_at = models.DateTimeField(null=True, blank=True)
 
     class Meta:
         permissions = LabPermission
@@ -163,6 +170,37 @@ class Meta:
     def __str__(self):
         return f"{self.name} ({self.principal_investigator_name}, {self.institution})"
 
+    def save(self, *args, **kwargs):
+        if self._state.adding and self.created_at is None:
+            self.created_at = dutimezone.now()
+        super().save(*args, **kwargs)
+
+    def get_response_stats(self) -> dict:
+        """Return tallied response counts for this lab, broken down by study type and time range.
+
+        Uses effective_is_tallied logic: researcher override takes precedence over is_tallied.
+        """
+        one_year_ago = dutimezone.now() - timedelta(days=365)
+
+        # StudyType id=2 is external (consistent with StudyType.is_external)
+        base_qs = Response.objects.filter(study__lab=self).filter(EFFECTIVELY_TALLIED_Q)
+        internal_qs = base_qs.exclude(study__study_type__id=2)
+        external_qs = base_qs.filter(study__study_type__id=2)
+
+        internal_all = internal_qs.count()
+        internal_year = internal_qs.filter(date_created__gte=one_year_ago).count()
+        external_all = external_qs.count()
+        external_year = external_qs.filter(date_created__gte=one_year_ago).count()
+
+        return {
+            "internal_all_time": internal_all,
+            "internal_last_year": internal_year,
+            "external_all_time": external_all,
+            "external_last_year": external_year,
+            "total_all_time": internal_all + external_all,
+            "total_last_year": internal_year + external_year,
+        }
+
 
 # Using Direct foreign keys for guardian, see:
 # https://django-guardian.readthedocs.io/en/stable/userguide/performance.html
@@ -566,11 +604,7 @@ def tallied_response_count(self) -> int:
         Returns:
             int: Count of tallied responses
         """
-        is_effectively_tallied = models.Q(
-            is_tallied_researcher_override__isnull=False,
-            is_tallied_researcher_override=True,
-        ) | models.Q(is_tallied_researcher_override__isnull=True, is_tallied=True)
-        return self.responses.filter(is_effectively_tallied).count()
+        return self.responses.filter(EFFECTIVELY_TALLIED_Q).count()
 
     @property
     def has_reached_max_responses(self) -> bool:

studies/queries.py

@@ -24,6 +24,7 @@
 from attachment_helpers import get_url
 from studies.models import (
     ACCEPTED,
+    EFFECTIVELY_TALLIED_Q,
     PENDING,
     REJECTED,
     ConsentRuling,
@@ -290,18 +291,14 @@ def get_response_type_counts_external(study) -> Dict:
     - tallied: all effectively tallied non-preview responses
     - total: all responses (preview/tallied/untallied)
     """
-    is_effectively_tallied = Q(
-        is_tallied_researcher_override__isnull=False,
-        is_tallied_researcher_override=True,
-    ) | Q(is_tallied_researcher_override__isnull=True, is_tallied=True)
     is_effectively_untallied = Q(
         is_tallied_researcher_override__isnull=False,
         is_tallied_researcher_override=False,
     ) | Q(is_tallied_researcher_override__isnull=True, is_tallied=False)
 
     counts = study.responses.aggregate(
         preview=Count("id", filter=Q(is_preview=True)),
-        tallied=Count("id", filter=Q(is_preview=False) & is_effectively_tallied),
+        tallied=Count("id", filter=Q(is_preview=False) & EFFECTIVELY_TALLIED_Q),
         untallied=Count("id", filter=Q(is_preview=False) & is_effectively_untallied),
     )
     counts["total"] = counts["preview"] + counts["tallied"] + counts["untallied"]
@@ -330,10 +327,6 @@ def get_response_type_counts(study) -> Dict:
     - total_rejected: all responses with rejected consent (preview_rejected + nonpreview_rejected)
     - total: total_available + total_pending + total_rejected
     """
-    is_effectively_tallied = Q(
-        is_tallied_researcher_override__isnull=False,
-        is_tallied_researcher_override=True,
-    ) | Q(is_tallied_researcher_override__isnull=True, is_tallied=True)
     is_effectively_untallied = Q(
         is_tallied_researcher_override__isnull=False,
         is_tallied_researcher_override=False,
@@ -355,12 +348,12 @@ def get_response_type_counts(study) -> Dict:
         tallied_approved=Count(
             "id",
             filter=Q(is_preview=False)
-            & is_effectively_tallied
+            & EFFECTIVELY_TALLIED_Q
             & Q(current_ruling="accepted"),
         ),
         tallied_pending=Count(
             "id",
-            filter=Q(is_preview=False) & is_effectively_tallied & is_pending_ruling,
+            filter=Q(is_preview=False) & EFFECTIVELY_TALLIED_Q & is_pending_ruling,
         ),
         untallied_approved=Count(
             "id",

studies/tasks.py

@@ -179,6 +179,14 @@ def _deserialized(user_grouped_targets, number_of_parents: int = 100):
 def _validated(deserialized_groups):
     """Yield only groups with targets that satisfy criteria for their respective studies."""
     for user, child_study_pairs in deserialized_groups:
+        if not user or not user.username or "@" not in user.username:
+            # Skip users that were deleted between the SQL query and ORM cache lookup,
+            # or who have no email address (which would break unsubscribe URL generation).
+            logger.warning(
+                "Skipping announcement email target: user %s has no valid username.",
+                getattr(user, "id", None),
+            )
+            continue
         valid_message_targets = []
         for pair in child_study_pairs:
             child, study = pair