diff --git a/.gitignore b/.gitignore index 9bd7661b3..a2948fbf3 100644 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,8 @@ data bootstrap-5* .python-version + +.aws/* +.claude/ +scratch/* +aws-lambda/* diff --git a/accounts/models.py b/accounts/models.py index 911433ab2..d732e35c2 100644 --- a/accounts/models.py +++ b/accounts/models.py @@ -1,9 +1,12 @@ import base64 import hashlib +import logging import uuid from io import BytesIO from typing import Union +logger = logging.getLogger(__name__) + import pydenticon import pyotp from bitfield import BitField @@ -681,6 +684,12 @@ def send_as_email(self): recipient_email_list = list(self.recipients.values_list("username", flat=True)) for to_email in recipient_email_list: + if not to_email or "@" not in to_email: + logger.warning( + "Skipping custom email recipient: username %s is not a valid email address.", + to_email, + ) + continue user = User.objects.get(username=to_email) context.update(token=user.generate_token(), username=to_email) send_mail.delay( diff --git a/exp/views/lab.py b/exp/views/lab.py index 53c91c826..0aa7bf837 100644 --- a/exp/views/lab.py +++ b/exp/views/lab.py @@ -73,6 +73,7 @@ def get_context_data(self, **kwargs): context["custom_url"] = self.request.build_absolute_uri( reverse("web:lab-studies-list", args=[lab.slug]) ) + context["lab_stats"] = lab.get_response_stats() return context diff --git a/pyproject.toml b/pyproject.toml index ff8209be4..06aa2a479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.13.*" dependencies = [ "boto3==1.40.52", "ciso8601==2.3.2", - "django==5.2.13", + "django==5.2.14", "django-ace-overlay", "django-bitfield==2.2.0", "django-bootstrap-icons==0.9.0", diff --git a/studies/forms.py b/studies/forms.py index 104529022..7dba801c8 100644 --- a/studies/forms.py +++ b/studies/forms.py @@ -120,6 +120,15 @@ class Meta: "badge": "This image will be shown at the top of your custom URL page when it is viewed on a mobile device/narrow browser window, and as a badge/avatar image for your lab. This image should be square. Please keep your file size less than 1 MB.", } + def clean_description(self): + """Lazy, on-demand fix for malformed lab description values in the database that contain literal \r\n strings.""" + description = self.cleaned_data.get("description", "") + return ( + description.replace("\\r\\n", "\n") + .replace("\\r", "\n") + .replace("\\n", "\n") + ) + def clean_banner(self): cleaned_banner = self.cleaned_data["banner"] ratio = 2 diff --git a/studies/management/commands/delete_lab.py b/studies/management/commands/delete_lab.py new file mode 100644 index 000000000..d320cd36f --- /dev/null +++ b/studies/management/commands/delete_lab.py @@ -0,0 +1,153 @@ +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction +from guardian.models import GroupObjectPermission, UserObjectPermission + +from studies.models import Lab + +EXPECTED_RELS = { + ("Study", "lab"), + ("LabUserObjectPermission", "content_object"), + ("LabGroupObjectPermission", "content_object"), +} + + +class Command(BaseCommand): + help = "Safely delete a Lab and its associated permissions and groups." + + def add_arguments(self, parser): + parser.add_argument( + "--uuid", type=str, required=True, help="UUID of the Lab to delete." + ) + parser.add_argument( + "--keep-groups", + action="store_true", + help="Preserve the Lab's auth Groups after deletion.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview deletion without deleting.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Actually perform deletion.", + ) + + def handle(self, *args, **options): + uuid = options["uuid"] + keep_groups = options["keep_groups"] + dry_run = options["dry_run"] + force = options["force"] + + if not dry_run and not force: + raise CommandError( + "Refusing to delete without --force. Use --dry-run to preview." + ) + + self.stdout.write("Inspecting relations pointing to Lab...\n") + found_rels = set() + for rel in Lab._meta.related_objects: + model_name = rel.related_model.__name__ + field_name = rel.field.name + on_delete = rel.field.remote_field.on_delete.__name__ + nullable = rel.field.null + found_rels.add((model_name, field_name)) + self.stdout.write( + f"- {model_name}.{field_name} (on_delete={on_delete}, nullable={nullable})" + ) + + unexpected = found_rels - EXPECTED_RELS + if unexpected: + raise CommandError( + f"Refusing to delete Lab. Unexpected relations detected: {unexpected}" + ) + + try: + lab = Lab.objects.get(uuid=uuid) + except (Lab.DoesNotExist, ValidationError): + raise CommandError(f"Lab with UUID {uuid} is invalid or not found.") + + self.stdout.write(self.style.WARNING(f"\nLab: {lab.name} ({lab.uuid})")) + + study_count = lab.studies.count() + if study_count > 0: + raise CommandError( + f"Refusing to delete lab: {study_count} studies are still attached." + ) + + lab_ct = ContentType.objects.get_for_model(Lab) + user_perm_count = UserObjectPermission.objects.filter( + content_type=lab_ct, object_pk=str(lab.pk) + ).count() + group_perm_count = GroupObjectPermission.objects.filter( + content_type=lab_ct, object_pk=str(lab.pk) + ).count() + researcher_count = lab.researchers.count() + requested_count = lab.requested_researchers.count() + groups = [ + g + for g in [ + lab.guest_group, + lab.readonly_group, + lab.member_group, + lab.admin_group, + ] + if g is not None + ] + + self.stdout.write(f" User object permissions to delete: {user_perm_count}") + self.stdout.write(f" Group object permissions to delete: {group_perm_count}") + self.stdout.write(f" Researchers to remove: {researcher_count}") + self.stdout.write(f" Pending join requests to remove: {requested_count}") + self.stdout.write( + f" Auth groups to {'preserve' if keep_groups else 'delete'}: {len(groups)}" + ) + self.stdout.write(f" Badge image to delete: {lab.badge.name or 'none'}") + self.stdout.write(f" Banner image to delete: {lab.banner.name or 'none'}") + + if dry_run: + self.stdout.write( + self.style.SUCCESS("\nDry run complete. No deletion performed.") + ) + return + + with transaction.atomic(): + lab = Lab.objects.select_for_update().get(uuid=uuid) + + UserObjectPermission.objects.filter( + content_type=lab_ct, object_pk=str(lab.pk) + ).delete() + GroupObjectPermission.objects.filter( + content_type=lab_ct, object_pk=str(lab.pk) + ).delete() + + lab.researchers.clear() + lab.requested_researchers.clear() + + if lab.badge: + lab.badge.delete(save=False) + if lab.banner: + lab.banner.delete(save=False) + + groups = [ + g + for g in [ + lab.guest_group, + lab.readonly_group, + lab.member_group, + lab.admin_group, + ] + if g is not None + ] + lab.delete() + self.stdout.write("Lab deleted.") + + if not keep_groups: + for group in groups: + self.stdout.write(f"Deleting group: {group.name}") + group.delete() + + self.stdout.write(self.style.SUCCESS("Done.")) diff --git a/studies/migrations/0107_add_lab_created_at.py b/studies/migrations/0107_add_lab_created_at.py new file mode 100644 index 000000000..873a55655 --- /dev/null +++ b/studies/migrations/0107_add_lab_created_at.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("studies", "0106_add_researcher_valid_response"), + ] + + operations = [ + migrations.AddField( + model_name="lab", + name="created_at", + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/studies/models.py b/studies/models.py index a168092eb..6d10bdea5 100644 --- a/studies/models.py +++ b/studies/models.py @@ -1,6 +1,6 @@ import logging import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from enum import Enum import boto3 @@ -54,6 +54,12 @@ REJECTED = "rejected" CONSENT_RULINGS = (ACCEPTED, REJECTED, PENDING) +# It would've made more sense to put this in queries.py and import it here, but queries.py imports from this file, so doing it the other way around avoids a circular import. +EFFECTIVELY_TALLIED_Q = models.Q( + is_tallied_researcher_override__isnull=False, + is_tallied_researcher_override=True, +) | models.Q(is_tallied_researcher_override__isnull=True, is_tallied=True) + S3_RESOURCE = boto3.resource("s3") S3_BUCKET = S3_RESOURCE.Bucket(settings.BUCKET_NAME) @@ -155,6 +161,7 @@ class Lab(models.Model): ) banner = models.ImageField(null=True, blank=True, upload_to="lab_images/") badge = models.ImageField(null=True, blank=True, upload_to="lab_images/") + created_at = models.DateTimeField(null=True, blank=True) class Meta: permissions = LabPermission @@ -163,6 +170,37 @@ class Meta: def __str__(self): return f"{self.name} ({self.principal_investigator_name}, {self.institution})" + def save(self, *args, **kwargs): + if self._state.adding and self.created_at is None: + self.created_at = dutimezone.now() + super().save(*args, **kwargs) + + def get_response_stats(self) -> dict: + """Return tallied response counts for this lab, broken down by study type and time range. + + Uses effective_is_tallied logic: researcher override takes precedence over is_tallied. + """ + one_year_ago = dutimezone.now() - timedelta(days=365) + + # StudyType id=2 is external (consistent with StudyType.is_external) + base_qs = Response.objects.filter(study__lab=self).filter(EFFECTIVELY_TALLIED_Q) + internal_qs = base_qs.exclude(study__study_type__id=2) + external_qs = base_qs.filter(study__study_type__id=2) + + internal_all = internal_qs.count() + internal_year = internal_qs.filter(date_created__gte=one_year_ago).count() + external_all = external_qs.count() + external_year = external_qs.filter(date_created__gte=one_year_ago).count() + + return { + "internal_all_time": internal_all, + "internal_last_year": internal_year, + "external_all_time": external_all, + "external_last_year": external_year, + "total_all_time": internal_all + external_all, + "total_last_year": internal_year + external_year, + } + # Using Direct foreign keys for guardian, see: # https://django-guardian.readthedocs.io/en/stable/userguide/performance.html @@ -566,11 +604,7 @@ def tallied_response_count(self) -> int: Returns: int: Count of tallied responses """ - is_effectively_tallied = models.Q( - is_tallied_researcher_override__isnull=False, - is_tallied_researcher_override=True, - ) | models.Q(is_tallied_researcher_override__isnull=True, is_tallied=True) - return self.responses.filter(is_effectively_tallied).count() + return self.responses.filter(EFFECTIVELY_TALLIED_Q).count() @property def has_reached_max_responses(self) -> bool: diff --git a/studies/queries.py b/studies/queries.py index 41cbd72ac..d049470f6 100644 --- a/studies/queries.py +++ b/studies/queries.py @@ -24,6 +24,7 @@ from attachment_helpers import get_url from studies.models import ( ACCEPTED, + EFFECTIVELY_TALLIED_Q, PENDING, REJECTED, ConsentRuling, @@ -290,10 +291,6 @@ def get_response_type_counts_external(study) -> Dict: - tallied: all effectively tallied non-preview responses - total: all responses (preview/tallied/untallied) """ - is_effectively_tallied = Q( - is_tallied_researcher_override__isnull=False, - is_tallied_researcher_override=True, - ) | Q(is_tallied_researcher_override__isnull=True, is_tallied=True) is_effectively_untallied = Q( is_tallied_researcher_override__isnull=False, is_tallied_researcher_override=False, @@ -301,7 +298,7 @@ def get_response_type_counts_external(study) -> Dict: counts = study.responses.aggregate( preview=Count("id", filter=Q(is_preview=True)), - tallied=Count("id", filter=Q(is_preview=False) & is_effectively_tallied), + tallied=Count("id", filter=Q(is_preview=False) & EFFECTIVELY_TALLIED_Q), untallied=Count("id", filter=Q(is_preview=False) & is_effectively_untallied), ) counts["total"] = counts["preview"] + counts["tallied"] + counts["untallied"] @@ -330,10 +327,6 @@ def get_response_type_counts(study) -> Dict: - total_rejected: all responses with rejected consent (preview_rejected + nonpreview_rejected) - total: total_available + total_pending + total_rejected """ - is_effectively_tallied = Q( - is_tallied_researcher_override__isnull=False, - is_tallied_researcher_override=True, - ) | Q(is_tallied_researcher_override__isnull=True, is_tallied=True) is_effectively_untallied = Q( is_tallied_researcher_override__isnull=False, is_tallied_researcher_override=False, @@ -355,12 +348,12 @@ def get_response_type_counts(study) -> Dict: tallied_approved=Count( "id", filter=Q(is_preview=False) - & is_effectively_tallied + & EFFECTIVELY_TALLIED_Q & Q(current_ruling="accepted"), ), tallied_pending=Count( "id", - filter=Q(is_preview=False) & is_effectively_tallied & is_pending_ruling, + filter=Q(is_preview=False) & EFFECTIVELY_TALLIED_Q & is_pending_ruling, ), untallied_approved=Count( "id", diff --git a/studies/tasks.py b/studies/tasks.py index 99bfc8e77..bbf796283 100644 --- a/studies/tasks.py +++ b/studies/tasks.py @@ -179,6 +179,14 @@ def _deserialized(user_grouped_targets, number_of_parents: int = 100): def _validated(deserialized_groups): """Yield only groups with targets that satisfy criteria for their respective studies.""" for user, child_study_pairs in deserialized_groups: + if not user or not user.username or "@" not in user.username: + # Skip users that were deleted between the SQL query and ORM cache lookup, + # or who have no email address (which would break unsubscribe URL generation). + logger.warning( + "Skipping announcement email target: user %s has no valid username.", + getattr(user, "id", None), + ) + continue valid_message_targets = [] for pair in child_study_pairs: child, study = pair diff --git a/studies/templates/studies/lab_detail.html b/studies/templates/studies/lab_detail.html index b7f3cd103..88842f0c4 100644 --- a/studies/templates/studies/lab_detail.html +++ b/studies/templates/studies/lab_detail.html @@ -28,66 +28,92 @@ {% endif %} {% set_variable link_edit_lab|add:link_lab_members|add:lab_join_message as page_title_right_content %} {% page_title lab.name right_side_elements=page_title_right_content %} -
{{ lab.institution }}
-{{ lab.principal_investigator_name }}
-{{ lab.contact_email }}
-{{ lab.contact_phone }}
-- {{ custom_url }} -
-{{ lab.irb_contact_info }}
-{{ lab.institution }}
+{{ lab.principal_investigator_name }}
+{{ lab.contact_email }}
+{{ lab.contact_phone }}
++ {{ custom_url }} +
+{{ lab.description|squash_spaces|linebreaks }}
+{{ lab.irb_contact_info }}
+| + | Internal | +External | +Total | +
|---|---|---|---|
| Last 12 months | +{{ lab_stats.internal_last_year }} | +{{ lab_stats.external_last_year }} | +{{ lab_stats.total_last_year }} | +
| + {% if lab.created_at %} + Since {{ lab.created_at|date:"N j, Y" }} + {% else %} + All time + {% endif %} | +{{ lab_stats.internal_all_time }} | +{{ lab_stats.external_all_time }} | +{{ lab_stats.total_all_time }} | +
Note: Session counts reflect the number of “tallied” sessions across all of your studies, including only non-preview responses from eligible children. For internal studies only, tallied sessions are also limited to those which are marked complete and have a consent status of approved or pending. See documentation for additional detail on session tallies and how to submit individual session updates.
+