From ad74a05219d7c8323d583fd055ca1887f0509520 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Wed, 2 Sep 2026 12:11:01 -0500 Subject: [PATCH 01/13] Capture usage metrics for API functions including query params --- app/api/views.py | 12 ++++ app/host/decorators.py | 15 +++-- ...4_usagemetricslog_query_params_and_more.py | 63 +++++++++++++++++++ app/host/models.py | 6 +- 4 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 app/host/migrations/0054_usagemetricslog_query_params_and_more.py diff --git a/app/api/views.py b/app/api/views.py index aa37c0de..9263ca6e 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -7,6 +7,7 @@ from django.conf import settings from django.http import StreamingHttpResponse from django.http import JsonResponse +from django.utils.decorators import method_decorator from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiResponse, OpenApiParameter from rest_framework import status @@ -147,6 +148,7 @@ class Meta: ############################################################ # ViewSets +@method_decorator(log_usage_metric(), name="dispatch") class TransientViewSet(viewsets.ReadOnlyModelViewSet): queryset = Transient.objects.all() serializer_class = TransientSerializer @@ -154,6 +156,7 @@ class TransientViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = TransientFilter +@method_decorator(log_usage_metric(), name="dispatch") class ApertureViewSet(viewsets.ReadOnlyModelViewSet): queryset = Aperture.objects.all() serializer_class = ApertureSerializer @@ -161,6 +164,7 @@ class ApertureViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = ApertureFilter +@method_decorator(log_usage_metric(), name="dispatch") class CutoutViewSet(viewsets.ReadOnlyModelViewSet): queryset = Cutout.objects.all() serializer_class = CutoutSerializer @@ -173,12 +177,14 @@ class CutoutViewSet(viewsets.ReadOnlyModelViewSet): 404: OpenApiResponse(description="File not found"), } ) + @method_decorator(log_usage_metric(), name="dispatch") @action(methods=['get'], detail=True, url_path="download") def download(self, request, pk=None): cutout = self.get_object() return stream_download_file(cutout.fits.name) +@method_decorator(log_usage_metric(), name="dispatch") class FilterViewSet(viewsets.ReadOnlyModelViewSet): queryset = Filter.objects.all() serializer_class = FilterSerializer @@ -186,6 +192,7 @@ class FilterViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = FilterFilter +@method_decorator(log_usage_metric(), name="dispatch") class AperturePhotometryViewSet(viewsets.ReadOnlyModelViewSet): queryset = AperturePhotometry.objects.all() serializer_class = AperturePhotometrySerializer @@ -193,6 +200,7 @@ class AperturePhotometryViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = AperturePhotometryFilter +@method_decorator(log_usage_metric(), name="dispatch") class SEDFittingResultViewSet(viewsets.ReadOnlyModelViewSet): queryset = SEDFittingResult.objects.all() serializer_class = SEDFittingResultSerializer @@ -215,6 +223,7 @@ class SEDFittingResultViewSet(viewsets.ReadOnlyModelViewSet): 400: OpenApiResponse(description="Unknown file type"), } ) + @method_decorator(log_usage_metric(), name="dispatch") @action(methods=['get'], detail=True, url_path=r"download/(?P[^/.]+)") def download(self, request, pk=None, file_type: str = None): if file_type not in self.allowed_file_types: @@ -224,6 +233,7 @@ def download(self, request, pk=None, file_type: str = None): return stream_download_file(file_field.name) +@method_decorator(log_usage_metric(), name="dispatch") class TaskRegisterViewSet(viewsets.ReadOnlyModelViewSet): queryset = TaskRegister.objects.all() serializer_class = TaskRegisterSerializer @@ -231,11 +241,13 @@ class TaskRegisterViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = TaskRegisterFilter +@method_decorator(log_usage_metric(), name="dispatch") class TaskViewSet(viewsets.ReadOnlyModelViewSet): queryset = Task.objects.all() serializer_class = TaskSerializer +@method_decorator(log_usage_metric(), name="dispatch") class HostViewSet(viewsets.ReadOnlyModelViewSet): queryset = Host.objects.all() serializer_class = HostSerializer diff --git a/app/host/decorators.py b/app/host/decorators.py index fdff3d47..aad9f8b3 100644 --- a/app/host/decorators.py +++ b/app/host/decorators.py @@ -6,7 +6,6 @@ # from .models import ExternalResourceCall from .models import UsageMetricsLog -import json from textwrap import shorten # def log_resource_call(resource_name): @@ -38,12 +37,7 @@ def log_usage_metric(): - """ - Decorator to log a usage metric based on the request. - - Returns: - Decorator function. - """ + """Decorator to log a usage metric based on the request""" def decorator_save(func): @functools.wraps(func) def wrapper_save(*args, **kwargs): @@ -66,7 +60,11 @@ def wrapper_save(*args, **kwargs): if 'full_info' in post_data: full_info = re.split(r'\r\n|\n|\r', post_data['full_info']) post_data['full_info'] = full_info - submitted_data = json.dumps(post_data) + submitted_data = post_data + try: + query_params = request.GET + except AttributeError: + query_params = {} # Create and save the data to a new usage metric log object try: request_ip = request.META["REMOTE_ADDR"] @@ -81,6 +79,7 @@ def wrapper_save(*args, **kwargs): request_method=shorten(request.method, width=10, placeholder="..."), request_time=timezone.now(), submitted_data=submitted_data, + query_params=query_params, request_user=request.user.username[:150], request_ip=request_ip, request_user_agent=request_user_agent, diff --git a/app/host/migrations/0054_usagemetricslog_query_params_and_more.py b/app/host/migrations/0054_usagemetricslog_query_params_and_more.py new file mode 100644 index 00000000..5f103462 --- /dev/null +++ b/app/host/migrations/0054_usagemetricslog_query_params_and_more.py @@ -0,0 +1,63 @@ +"""UsageMetricsLog model migration script + +Add a new JSONField query_params field to the the UsageMetricsLog model and convert the submitted_data field +from TextField to JSONField. +""" + +from django.db import migrations, models +import json + + +def convert_submitted_data(apps, schema_editor): + UsageMetricsLog = apps.get_model("host", "UsageMetricsLog") + # Use .iterator() for more efficient one-pass processing of large tables + for record in UsageMetricsLog.objects.all().iterator(): + raw_value = record.submitted_data + if raw_value is None or raw_value.strip() == "": + record.submitted_data_json = None + # Using update_fields= performs a smaller UPDATE operation + record.save(update_fields=["submitted_data_json"]) + continue + try: + parsed_value = json.loads(raw_value) + except (TypeError, ValueError) as exc: + # !r tells an f-string to format the value using its repr() representation instead of its normal str() + # representation, useful in error messages because it makes invisible or ambiguous characters visible. + raise RuntimeError(f"UsageMetricsLog {record.pk} contains invalid JSON: {raw_value!r}") from exc + record.submitted_data_json = parsed_value + record.save(update_fields=["submitted_data_json"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('host', '0053_host_spectrum_task'), + ] + + operations = [ + migrations.AddField( + model_name="usagemetricslog", + name="submitted_data_json", + field=models.JSONField(null=True, blank=True), + ), + migrations.RunPython(convert_submitted_data), + migrations.RemoveField( + model_name="usagemetricslog", + name="submitted_data", + ), + migrations.RenameField( + model_name="usagemetricslog", + old_name="submitted_data_json", + new_name="submitted_data", + ), + migrations.AlterField( + model_name='usagemetricslog', + name='submitted_data', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AddField( + model_name='usagemetricslog', + name='query_params', + field=models.JSONField(blank=True, default=dict), + ), + ] diff --git a/app/host/models.py b/app/host/models.py index eeace033..d69993b3 100644 --- a/app/host/models.py +++ b/app/host/models.py @@ -672,14 +672,16 @@ class UsageMetricsLog(models.Model): request_url (models.CharField): The requested URL request_method (models.CharField): The HTTP method of the request request_time (models.DateTimeField): Time of request. - submitted_data (models.TextField): The data submitted in the request + submitted_data (models.JSONField): The data submitted in the request + query_params (models.JSONField): The query parameters submitted in the request request_user (models.CharField): The user that made the request (if authenticated). request_ip (models.CharField): The source IP that made the request. """ request_url = models.CharField(max_length=100, blank=False) request_method = models.CharField(max_length=10, blank=False) request_time = models.DateTimeField(auto_now_add=True, blank=False) - submitted_data = models.TextField(blank=True, default='') + submitted_data = models.JSONField(blank=True, default=dict) + query_params = models.JSONField(blank=True, default=dict) request_user = models.CharField(max_length=150, blank=False) request_ip = models.CharField(max_length=45, blank=True, default='') request_user_agent = models.CharField(max_length=400, blank=True, default='') From cb4a956d46b3e0987c91e52db85dcd6c07602441 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Thu, 3 Sep 2026 10:00:50 -0500 Subject: [PATCH 02/13] Log usage of dataset API endpoints --- app/api/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/api/views.py b/app/api/views.py index 9263ca6e..f3412eac 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -437,6 +437,7 @@ def has_permission(self, request, view): return request.user.has_perm("host.delete_transient") +@method_decorator(log_usage_metric(), name="dispatch") class DatasetExportView(APIView): serializer_class = TransientDatasetSerializer @@ -539,7 +540,7 @@ def get(self, request, transient_name=''): return response -# TODO: add log_usage_metric decorator +@method_decorator(log_usage_metric(), name="dispatch") class DatasetView(APIView): def get_permissions(self): method = self.request.method From d05be6835c648af8a9fe5a07a1a098e2cf22d268 Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Thu, 3 Sep 2026 10:01:22 -0500 Subject: [PATCH 03/13] Open dataset JSON file in new tab --- app/host/templates/host/transient_actions.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/host/templates/host/transient_actions.html b/app/host/templates/host/transient_actions.html index a5cb4c73..e46ef1c4 100644 --- a/app/host/templates/host/transient_actions.html +++ b/app/host/templates/host/transient_actions.html @@ -10,7 +10,7 @@