diff --git a/CHANGELOG.md b/CHANGELOG.md index feaa1c57..d1224fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ Types of changes: - `Fixed`: for any bug fixes. - `Security`: in case of vulnerabilities. +## [2.1.0] + +### Changed + +- Improved the usage metrics logging system to handle class-based views, including the individual Blast data object endpoints and the transient dataset download endpoints. +- Replaced Alias API view functions with ModelViewSet subclass for consistency with other data object API endpoints and to benefit from the Django REST Framework. (While this API change is technically *not* backwards-compatible, the nature of the `/api/alias/` functionality and its lack of known use to date justifies this "minor" violation of semantic versioning.) + ## [2.0.1] ### Fixed diff --git a/app/api/serializers.py b/app/api/serializers.py index 5c191fd2..d4a8f391 100644 --- a/app/api/serializers.py +++ b/app/api/serializers.py @@ -138,15 +138,13 @@ class Meta: class AliasSerializer(serializers.ModelSerializer): - transient = TransientSerializer(read_only=True) - host = HostSerializer(read_only=True) + transient = serializers.SerializerMethodField() + host = serializers.SerializerMethodField() class Meta: model = models.Alias fields = ["alias", "transient", "host"] - - transient = serializers.SerializerMethodField() - host = serializers.SerializerMethodField() + depth = 1 @extend_schema_field(serializers.CharField(allow_null=True)) def get_transient(self, obj): diff --git a/app/api/tests/test_api.py b/app/api/tests/test_api.py index 5da83206..5255ca12 100644 --- a/app/api/tests/test_api.py +++ b/app/api/tests/test_api.py @@ -92,9 +92,10 @@ def test_alias(self): name = '2022testone' alias = '2022testone-alias-test!' # Attempt to create an alias without permission - response = self.client.post(f'/api/alias/{alias}/{object_type}/{name}/') - # print(f'[{response.status_code}] {response.content}') - data = json.loads(response.content) + response = self.client.post('/api/alias/', json={ + 'alias': alias, + object_type: name, + }) self.assertTrue(response.status_code == status.HTTP_403_FORBIDDEN) # Grant the user permission add_permission = Permission.objects.get( @@ -103,10 +104,11 @@ def test_alias(self): ) user.user_permissions.add(add_permission) assert user.has_perm('host.add_alias') - response = self.client.post(f'/api/alias/{alias}/{object_type}/{name}/') + response = self.client.post('/api/alias/', data={ + 'alias': alias, + object_type: name, + }) self.assertTrue(response.status_code == status.HTTP_201_CREATED) - data = json.loads(response.content) - self.assertTrue(data["message"].startswith("Alias successfully created:")) # Fetch information about the alias anonymously self.client.logout() response = self.client.get(f'/api/alias/{alias}/') @@ -114,11 +116,13 @@ def test_alias(self): # Fail when attempting to create another alias with the same name self.client.force_login(user) object_type = 'host' - response = self.client.post(f'/api/alias/{alias}/{object_type}/{name}/') - self.assertTrue(response.status_code == status.HTTP_409_CONFLICT) + response = self.client.post('/api/alias/', data={ + 'alias': alias, + object_type: name, + }) + self.assertTrue(response.status_code == status.HTTP_400_BAD_REQUEST) # Attempt to delete an alias without permission response = self.client.delete(f'/api/alias/{alias}/') - # print(f'[{response.status_code}] {response.content}') self.assertTrue(response.status_code == status.HTTP_403_FORBIDDEN) # Grant the user delete permission delete_permission = Permission.objects.get( @@ -128,10 +132,8 @@ def test_alias(self): user.user_permissions.add(delete_permission) # Delete the alias response = self.client.delete(f'/api/alias/{alias}/') - # print(f'[{response.status_code}] {response.content}') self.assertTrue(response.status_code == status.HTTP_204_NO_CONTENT) # Attempt to delete a non-existent alias alias = 'foo' response = self.client.delete(f'/api/alias/{alias}/') self.assertTrue(response.status_code == status.HTTP_404_NOT_FOUND) - # print(f'[{response.status_code}] {response.content}') diff --git a/app/api/urls.py b/app/api/urls.py index 2b1ac6e8..ec634f2c 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -1,8 +1,9 @@ import os -from django.urls import path, re_path +from django.urls import re_path, include +from rest_framework.routers import DefaultRouter -from . import views +import api.views base_path = os.environ.get("BASE_PATH", "").strip("/") if base_path != "": @@ -11,20 +12,30 @@ urlpatterns = [ re_path( base_path + r"^dataset/(?P[a-zA-Z0-9_-]+)/export/$", - views.DatasetExportView.as_view(), + api.views.DatasetExportView.as_view(), ), re_path( base_path + r"^dataset/(?P[a-zA-Z0-9_-]+)/$", - views.DatasetView.as_view(), + api.views.DatasetView.as_view(), ), - path(base_path + 'alias//', views.alias_handler_get_delete, ), - path(base_path + 'alias////', views.alias_handler_post), ] -# if os.environ.get("ALLOW_API_POST") == "YES": -# urlpatterns.append( -# path( -# f"""{base_path}transient/post/name=&ra=&dec=""", -# views.post_transient, -# ) -# ) +router = DefaultRouter() + +router.register(r"transient", api.views.TransientViewSet) +router.register(r"aperture", api.views.ApertureViewSet) +router.register(r"cutout", api.views.CutoutViewSet, basename="cutout") +router.register(r"filter", api.views.FilterViewSet) +router.register(r"aperturephotometry", api.views.AperturePhotometryViewSet) +router.register(r"sedfittingresult", api.views.SEDFittingResultViewSet, basename="sedfittingresult") +router.register(r"taskregister", api.views.TaskRegisterViewSet) +router.register(r"task", api.views.TaskViewSet) +router.register(r"host", api.views.HostViewSet) +router.register(r"alias", api.views.AliasViewSet) + +# Login/Logout +api_url_patterns = [ + re_path("", include(router.urls)), +] + +urlpatterns += api_url_patterns diff --git a/app/api/views.py b/app/api/views.py index aa37c0de..7e6d9526 100644 --- a/app/api/views.py +++ b/app/api/views.py @@ -7,14 +7,17 @@ from django.conf import settings from django.http import StreamingHttpResponse from django.http import JsonResponse +from django.utils.decorators import method_decorator from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiResponse, OpenApiParameter +from drf_spectacular.utils import extend_schema, OpenApiResponse, OpenApiParameter from rest_framework import status +from rest_framework import serializers from rest_framework import viewsets from rest_framework.views import APIView -from rest_framework.decorators import api_view, action +from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.permissions import BasePermission +from rest_framework.exceptions import NotFound from host.object_store import ObjectStore from host.models import Aperture from host.models import AperturePhotometry @@ -59,18 +62,27 @@ def stream_download_file(file_path): ############################################################ # Filter Sets class TransientFilter(django_filters.FilterSet): + name = django_filters.CharFilter( + field_name="name", lookup_expr="exact", + help_text="Find specific transient by name") redshift_lte = django_filters.NumberFilter( - field_name="redshift", lookup_expr="lte") + field_name="redshift", lookup_expr="lte", + help_text="Find transients with redshift less than value") redshift_gte = django_filters.NumberFilter( - field_name="redshift", lookup_expr="gte") + field_name="redshift", lookup_expr="gte", + help_text="Find transients with redshift greater than value") host_redshift_lte = django_filters.NumberFilter( - field_name="host__redshift", lookup_expr="lte") + field_name="host__redshift", lookup_expr="lte", + help_text="Find transients with host redshift less than value") host_redshift_gte = django_filters.NumberFilter( - field_name="host__redshift", lookup_expr="gte") + field_name="host__redshift", lookup_expr="gte", + help_text="Find transients with host redshift greater than value") host_photometric_redshift_lte = django_filters.NumberFilter( - field_name="host__photometric_redshift", lookup_expr="lte") + field_name="host__photometric_redshift", lookup_expr="lte", + help_text="Find transients with host photometric redshift less than value") host_photometric_redshift_gte = django_filters.NumberFilter( - field_name="host__photometric_redshift", lookup_expr="gte") + field_name="host__photometric_redshift", lookup_expr="gte", + help_text="Find transients with host photometric redshift greater than value") class Meta: model = Transient @@ -94,6 +106,16 @@ class Meta: fields = ("name",) +class AliasFilter(django_filters.FilterSet): + alias = django_filters.Filter(field_name="alias") + transient = django_filters.Filter(field_name="transient") + host = django_filters.Filter(field_name="host") + + class Meta: + model = Alias + fields = () + + class ApertureFilter(django_filters.FilterSet): transient = django_filters.Filter(field_name="transient__name") @@ -147,6 +169,11 @@ class Meta: ############################################################ # ViewSets +@method_decorator(log_usage_metric(), name="dispatch") +@extend_schema( + summary="Transient search", + # description=dedent('''''') +) class TransientViewSet(viewsets.ReadOnlyModelViewSet): queryset = Transient.objects.all() serializer_class = TransientSerializer @@ -154,6 +181,7 @@ class TransientViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = TransientFilter +@method_decorator(log_usage_metric(), name="dispatch") class ApertureViewSet(viewsets.ReadOnlyModelViewSet): queryset = Aperture.objects.all() serializer_class = ApertureSerializer @@ -161,6 +189,7 @@ class ApertureViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = ApertureFilter +@method_decorator(log_usage_metric(), name="dispatch") class CutoutViewSet(viewsets.ReadOnlyModelViewSet): queryset = Cutout.objects.all() serializer_class = CutoutSerializer @@ -173,12 +202,14 @@ class CutoutViewSet(viewsets.ReadOnlyModelViewSet): 404: OpenApiResponse(description="File not found"), } ) + @method_decorator(log_usage_metric(), name="dispatch") @action(methods=['get'], detail=True, url_path="download") def download(self, request, pk=None): cutout = self.get_object() return stream_download_file(cutout.fits.name) +@method_decorator(log_usage_metric(), name="dispatch") class FilterViewSet(viewsets.ReadOnlyModelViewSet): queryset = Filter.objects.all() serializer_class = FilterSerializer @@ -186,6 +217,7 @@ class FilterViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = FilterFilter +@method_decorator(log_usage_metric(), name="dispatch") class AperturePhotometryViewSet(viewsets.ReadOnlyModelViewSet): queryset = AperturePhotometry.objects.all() serializer_class = AperturePhotometrySerializer @@ -193,6 +225,7 @@ class AperturePhotometryViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = AperturePhotometryFilter +@method_decorator(log_usage_metric(), name="dispatch") class SEDFittingResultViewSet(viewsets.ReadOnlyModelViewSet): queryset = SEDFittingResult.objects.all() serializer_class = SEDFittingResultSerializer @@ -215,6 +248,7 @@ class SEDFittingResultViewSet(viewsets.ReadOnlyModelViewSet): 400: OpenApiResponse(description="Unknown file type"), } ) + @method_decorator(log_usage_metric(), name="dispatch") @action(methods=['get'], detail=True, url_path=r"download/(?P[^/.]+)") def download(self, request, pk=None, file_type: str = None): if file_type not in self.allowed_file_types: @@ -224,6 +258,7 @@ def download(self, request, pk=None, file_type: str = None): return stream_download_file(file_field.name) +@method_decorator(log_usage_metric(), name="dispatch") class TaskRegisterViewSet(viewsets.ReadOnlyModelViewSet): queryset = TaskRegister.objects.all() serializer_class = TaskRegisterSerializer @@ -231,11 +266,13 @@ class TaskRegisterViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = TaskRegisterFilter +@method_decorator(log_usage_metric(), name="dispatch") class TaskViewSet(viewsets.ReadOnlyModelViewSet): queryset = Task.objects.all() serializer_class = TaskSerializer +@method_decorator(log_usage_metric(), name="dispatch") class HostViewSet(viewsets.ReadOnlyModelViewSet): queryset = Host.objects.all() serializer_class = HostSerializer @@ -244,6 +281,73 @@ class HostViewSet(viewsets.ReadOnlyModelViewSet): lookup_value_regex = r"[^/]+[/]?" +@method_decorator(log_usage_metric(), name="dispatch") +class AliasViewSet(viewsets.ModelViewSet): + def get_permissions(self): + if self.request.method == "DELETE": + return [HasPermissionDeleteAlias()] + elif self.request.method == "POST": + return [HasPermissionCreateAlias()] + return [] + queryset = Alias.objects.select_related("transient", "host") + serializer_class = AliasSerializer + filter_backends = (DjangoFilterBackend,) + filterset_class = AliasFilter + + # Disable the PUT method + http_method_names = [ + "get", + "post", + "patch", + "delete", + "head", + "options", + ] + + # Create a serializer solely to support the drf_spectacular "extend_schema" decorator + # for the sake of OpenAPI spec generation. + class AliasCreateRequestSerializer(serializers.Serializer): + alias = serializers.CharField() + transient = serializers.CharField(required=False, allow_blank=False) + host = serializers.CharField(required=False, allow_blank=False) + + def validate(self, attrs): + transient = attrs.get("transient") + host = attrs.get("host") + if not transient and not host: + raise serializers.ValidationError("Provide either 'transient' or 'host'.") + return attrs + + @extend_schema( + request=AliasCreateRequestSerializer, + responses={ + 201: AliasSerializer, + }, + ) + def create(self, request, *args, **kwargs): + return super().create(request, *args, **kwargs) + + def perform_create(self, serializer): + transient_name = self.request.data.get("transient") + host_name = self.request.data.get("host") + + if bool(transient_name) == bool(host_name): + raise serializers.ValidationError("Specify EITHER 'transient' or 'host'.") + + if transient_name: + try: + transient = Transient.objects.get(name__exact=transient_name) + except Transient.DoesNotExist: + raise NotFound({"transient": f"No transient found with name '{transient_name}'."}) + serializer.save(transient=transient) + else: + try: + host = Host.objects.get(name__exact=host_name) + except Host.DoesNotExist: + raise NotFound({"host": f"No host found with name '{host_name}'."}) + serializer.save(host=host) + + def transient_exists(transient_name: str) -> bool: """ Checks if a transient exists in the database. @@ -306,111 +410,6 @@ def ra_dec_valid(ra: str, dec: str) -> bool: # status=status.HTTP_201_CREATED, # ) - -@extend_schema_view( - get=extend_schema( - parameters=[OpenApiParameter("alias", str, OpenApiParameter.PATH),], - request=None, - responses={ - 200: AliasSerializer, - 404: OpenApiResponse(description="Alias not found"), - } - ), - delete=extend_schema( - parameters=[OpenApiParameter("alias", str, OpenApiParameter.PATH),], - request=None, - responses={ - 204: OpenApiResponse(description="Alias deleted"), - 404: OpenApiResponse(description="Alias not found"), - } - ), -) -@api_view(["GET", "DELETE"]) -@log_usage_metric() -def alias_handler_get_delete(request, alias: str): - user_permissions = request.user.get_all_permissions() - if request.method == 'GET': - try: - alias = Alias.objects.get(alias__exact=alias) - return Response( - {"message": str(alias)}, - status=status.HTTP_200_OK, - ) - except Alias.DoesNotExist: - return Response( - {"message": f'''Alias with name "{alias}" does not exist.'''}, - status=status.HTTP_404_NOT_FOUND, - ) - if request.method == 'DELETE': - # Validate inputs - try: - assert alias - except AssertionError: - return Response(status=status.HTTP_400_BAD_REQUEST) - # Enforce authorization - if "host.delete_alias" not in user_permissions: - return Response( - {"message": "User does not have permissions to delete aliases"}, - status=status.HTTP_403_FORBIDDEN, - ) - try: - alias = Alias.objects.get(alias__exact=alias) - except Alias.DoesNotExist: - return Response(status=status.HTTP_404_NOT_FOUND) - alias.delete() - return Response(status=status.HTTP_204_NO_CONTENT) - - -@extend_schema_view( - post=extend_schema( - parameters=[OpenApiParameter("alias", str, OpenApiParameter.PATH),], - request=AliasSerializer, - responses={ - 201: AliasSerializer, - 409: OpenApiResponse(description="Alias already exists"), - } - ), -) -@api_view(["POST"]) -@log_usage_metric() -def alias_handler_post(request, alias: str, object_type: str = None, name: str = None): - user_permissions = request.user.get_all_permissions() - # Validate inputs - try: - assert object_type in ['transient', 'host'] - assert isinstance(name, str) and name - except AssertionError: - return Response({'message': 'Object type (transient or host) and name of object must be provided'}, - status=status.HTTP_400_BAD_REQUEST) - # Enforce authorization - if "host.add_alias" not in user_permissions: - return Response( - {"message": f"User does not have permissions to add aliases for {object_type}"}, - status=status.HTTP_403_FORBIDDEN, - ) - # Do not overwrite existing alias - if Alias.objects.filter(alias__exact=alias).exists(): - return Response( - {"message": f"{alias} is already in the database."}, - status=status.HTTP_409_CONFLICT - ) - try: - if object_type == 'transient': - target = Transient.objects.get(name__exact=name) - else: - target = Host.objects.get(name__exact=name) - except (Transient.DoesNotExist, Host.DoesNotExist): - return Response( - {"message": f"{object_type} with name {name} does not exist."}, - status=status.HTTP_404_NOT_FOUND, - ) - new_alias = Alias.objects.create(**{'alias': alias, object_type: target}) - return Response( - {"message": f"Alias successfully created: {str(new_alias)}"}, - status=status.HTTP_201_CREATED, - ) - - # TODO: Secure this endpoint with Django REST Framework permission_classes # @api_view(["PUT"]) # @permission_classes([IsAuthenticated]) @@ -420,11 +419,22 @@ def alias_handler_post(request, alias: str, object_type: str = None, name: str = # return Response({'message': f'Launched workflow for "{transient_name}": {result.task_id}'}) +class HasPermissionCreateAlias(BasePermission): + def has_permission(self, request, view): + return request.user.has_perm("host.add_alias") + + +class HasPermissionDeleteAlias(BasePermission): + def has_permission(self, request, view): + return request.user.has_perm("host.delete_alias") + + class HasPermissionDeleteTransient(BasePermission): def has_permission(self, request, view): return request.user.has_perm("host.delete_transient") +@method_decorator(log_usage_metric(), name="dispatch") class DatasetExportView(APIView): serializer_class = TransientDatasetSerializer @@ -527,7 +537,7 @@ def get(self, request, transient_name=''): return response -# TODO: add log_usage_metric decorator +@method_decorator(log_usage_metric(), name="dispatch") class DatasetView(APIView): def get_permissions(self): method = self.request.method diff --git a/app/app/settings.py b/app/app/settings.py index c4bb8548..1838b674 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -4,7 +4,7 @@ ###################################################################### # Blast application config # -APP_VERSION = '2.0.1' +APP_VERSION = '2.1.0' # Data paths DUSTMAPS_DATA_ROOT = os.environ.get("DUSTMAPS_DATA_ROOT", "/data/dustmaps") CUTOUT_ROOT = os.environ.get("CUTOUT_ROOT", "/data/cutout_cdn") @@ -250,7 +250,10 @@ 'DESCRIPTION': 'API Schema view for the Blast Transient Catalog', 'VERSION': '2.0.0', 'SERVE_INCLUDE_SCHEMA': False, - # OTHER SETTINGS + "SWAGGER_UI_SETTINGS": { + # Expand response/request model schemas to this depth + "defaultModelExpandDepth": 3, + }, } ###################################################################### diff --git a/app/host/decorators.py b/app/host/decorators.py index fdff3d47..a91c6aea 100644 --- a/app/host/decorators.py +++ b/app/host/decorators.py @@ -1,12 +1,12 @@ import functools import re +import json from django.utils import timezone from django.conf import settings # from .models import ExternalResourceCall from .models import UsageMetricsLog -import json from textwrap import shorten # def log_resource_call(resource_name): @@ -38,12 +38,7 @@ def log_usage_metric(): - """ - Decorator to log a usage metric based on the request. - - Returns: - Decorator function. - """ + """Decorator to log a usage metric based on the request""" def decorator_save(func): @functools.wraps(func) def wrapper_save(*args, **kwargs): @@ -56,7 +51,10 @@ def wrapper_save(*args, **kwargs): # Filter the submitted data object for POST requests submitted_data = '' if (request.method == "POST"): - post_data = {k: v for k, v in request.POST.copy().items() if v} + try: + post_data = json.loads(request.body.decode("utf-8")) + except Exception: + post_data = {k: v for k, v in request.POST.copy().items() if v} post_data.pop("csrfmiddlewaretoken", None) tns_names = [] if 'tns_names' in post_data: @@ -66,7 +64,11 @@ def wrapper_save(*args, **kwargs): if 'full_info' in post_data: full_info = re.split(r'\r\n|\n|\r', post_data['full_info']) post_data['full_info'] = full_info - submitted_data = json.dumps(post_data) + submitted_data = post_data + try: + query_params = request.GET + except AttributeError: + query_params = {} # Create and save the data to a new usage metric log object try: request_ip = request.META["REMOTE_ADDR"] @@ -81,6 +83,7 @@ def wrapper_save(*args, **kwargs): request_method=shorten(request.method, width=10, placeholder="..."), request_time=timezone.now(), submitted_data=submitted_data, + query_params=query_params, request_user=request.user.username[:150], request_ip=request_ip, request_user_agent=request_user_agent, diff --git a/app/host/migrations/0054_usagemetricslog_query_params_and_more.py b/app/host/migrations/0054_usagemetricslog_query_params_and_more.py new file mode 100644 index 00000000..bc58d37e --- /dev/null +++ b/app/host/migrations/0054_usagemetricslog_query_params_and_more.py @@ -0,0 +1,77 @@ +"""UsageMetricsLog and Alias model update migration script + +Add a new JSONField query_params field to the the UsageMetricsLog model and convert the submitted_data field +from TextField to JSONField. + +Replaces original Alias primary key with alias string itself. Should not need conversion logic because the +alias values are already unique. +""" + +from django.db import migrations, models +import json +import host.models + + +def convert_submitted_data(apps, schema_editor): + UsageMetricsLog = apps.get_model("host", "UsageMetricsLog") + # Use .iterator() for more efficient one-pass processing of large tables + for record in UsageMetricsLog.objects.all().iterator(): + raw_value = record.submitted_data + if raw_value is None or raw_value.strip() == "": + record.submitted_data_json = None + # Using update_fields= performs a smaller UPDATE operation + record.save(update_fields=["submitted_data_json"]) + continue + try: + parsed_value = json.loads(raw_value) + except (TypeError, ValueError) as exc: + # !r tells an f-string to format the value using its repr() representation instead of its normal str() + # representation, useful in error messages because it makes invisible or ambiguous characters visible. + raise RuntimeError(f"UsageMetricsLog {record.pk} contains invalid JSON: {raw_value!r}") from exc + record.submitted_data_json = parsed_value + record.save(update_fields=["submitted_data_json"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('host', '0053_host_spectrum_task'), + ] + + operations = [ + migrations.AddField( + model_name="usagemetricslog", + name="submitted_data_json", + field=models.JSONField(null=True, blank=True), + ), + migrations.RunPython(convert_submitted_data), + migrations.RemoveField( + model_name="usagemetricslog", + name="submitted_data", + ), + migrations.RenameField( + model_name="usagemetricslog", + old_name="submitted_data_json", + new_name="submitted_data", + ), + migrations.AlterField( + model_name='usagemetricslog', + name='submitted_data', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AddField( + model_name='usagemetricslog', + name='query_params', + field=models.JSONField(blank=True, default=dict), + ), + migrations.RemoveField( + model_name='alias', + name='id', + ), + migrations.AlterField( + model_name='alias', + name='alias', + field=models.CharField(max_length=64, primary_key=True, serialize=False, unique=True, + validators=[host.models.Alias.validate_name]), + ), + ] diff --git a/app/host/models.py b/app/host/models.py index eeace033..aee4f4d8 100644 --- a/app/host/models.py +++ b/app/host/models.py @@ -3,7 +3,7 @@ backend. """ import os - +from textwrap import dedent import pandas as pd from astropy import units as u from astropy.coordinates import SkyCoord @@ -41,8 +41,8 @@ class SkyObject(models.Model): of the host """ - ra_deg = models.FloatField() - dec_deg = models.FloatField() + ra_deg = models.FloatField(help_text='Right Ascension in decimal degrees e.g., 132.34564') + dec_deg = models.FloatField(help_text='Declination in decimal degrees e.g., 60.123424') class Meta: abstract = True @@ -87,16 +87,23 @@ class Host(SkyObject): """ name = models.CharField(max_length=100, blank=True, null=True, unique=True) - redshift = models.FloatField(null=True, blank=True) - redshift_err = models.FloatField(null=True, blank=True) - photometric_redshift = models.FloatField(null=True, blank=True) - photometric_redshift_err = models.FloatField(null=True, blank=True) - milkyway_dust_reddening = models.FloatField(null=True, blank=True) + redshift = models.FloatField(null=True, blank=True, + help_text='redshift e.g., 0.01') + # TODO: Clarify value range and example value + redshift_err = models.FloatField(null=True, blank=True, + help_text='redshift error') + photometric_redshift = models.FloatField(null=True, blank=True, + help_text='photometric redshift') + photometric_redshift_err = models.FloatField(null=True, blank=True, + help_text='photometric redshift error') + milkyway_dust_reddening = models.FloatField(null=True, blank=True, + help_text='transient E(B-V) e.g, 0.2') object_id = models.CharField(max_length=100, blank=True, null=True) catalog_name = models.CharField(max_length=100, blank=False, null=True) catalog_release = models.CharField(max_length=100, blank=False, null=True) objects = HostManager() - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') class Transient(SkyObject): @@ -146,24 +153,39 @@ def validate_name(name): raise ValidationError(f'''Invalid transient identifier: "{name}" may not contain consecutive ''' f'''"{nonconsecutive_char}" characters.''') - name = models.CharField(max_length=64, unique=True, validators=[validate_name]) + name = models.CharField(max_length=64, unique=True, validators=[validate_name], + help_text='unique transient name, e.g., 2022abc') display_name = models.CharField(null=True, blank=True) tns_id = models.IntegerField() tns_prefix = models.CharField(max_length=20) public_timestamp = models.DateTimeField(null=True, blank=True) - host = models.ForeignKey(Host, on_delete=models.SET_NULL, null=True, blank=True) + host = models.ForeignKey(Host, on_delete=models.SET_NULL, null=True, blank=True, + help_text='host associated with the transient') objects = TransientManager() tasks_initialized = models.CharField(max_length=20, default="False") redshift = models.FloatField(null=True, blank=True) - spectroscopic_class = models.CharField(max_length=20, null=True, blank=True) + spectroscopic_class = models.CharField(max_length=20, null=True, blank=True, + help_text='spectroscopic classification, if any') photometric_class = models.CharField(max_length=20, null=True, blank=True) milkyway_dust_reddening = models.FloatField(null=True, blank=True) - processing_status = models.CharField(max_length=20, default="processing") + processing_status = models.CharField( + max_length=20, + default="processing", + help_text=dedent('''processing status of the transient dataset. + * "processed" - transient has been complement processed and all data + should be present in the science payload. + * "processing" - transient is still processing this transient and + some parts of the science payload may not be populated at the current time. + * "blocked" - this transient has not been successfully fully processed + and some parts of the science payload will not be populated.''')) added_by = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) - progress = models.IntegerField(default=0) - software_version = models.CharField(max_length=50, blank=True, null=True) - update_comment = models.CharField(max_length=500, blank=True, null=True) - update_fields = models.CharField(max_length=500, blank=True, null=True) + progress = models.IntegerField(default=0, help_text='Percentage of the transient workflow that has been completed.') + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') + update_comment = models.CharField(max_length=500, blank=True, null=True, + help_text='A brief explanation for why the transient data were updated') + update_fields = models.CharField(max_length=500, blank=True, null=True, + help_text='Transient database fields that were last updated.') @property def best_redshift(self): @@ -428,7 +450,8 @@ class Cutout(models.Model): ) fits = models.FileField(upload_to=fits_file_path, null=True, blank=True) message = models.CharField(max_length=50, null=True, blank=True) - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') cropped = models.BooleanField(default=False, blank=True, null=False) # used if some downloads fail @@ -457,7 +480,8 @@ class Aperture(SkyObject): semi_major_axis_arcsec = models.FloatField() semi_minor_axis_arcsec = models.FloatField() type = models.CharField(max_length=20) - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') objects = ApertureManager() @@ -502,7 +526,8 @@ class AperturePhotometry(models.Model): magnitude = models.FloatField(blank=True, null=True) magnitude_error = models.FloatField(blank=True, null=True) is_validated = models.CharField(blank=True, null=True, max_length=40) - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') @property def flux_rounded(self): @@ -532,7 +557,8 @@ class StarFormationHistoryResult(models.Model): logsfr_tmin = models.FloatField(null=True, blank=True) logsfr_tmax = models.FloatField(null=True, blank=True) - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') def save(self, *args, **kwargs): self.software_version = settings.APP_VERSION @@ -620,7 +646,8 @@ class SEDFittingResult(models.Model): upload_to=npz_percentiles_file_path, null=True, blank=True ) model_file = models.FileField(upload_to=npz_model_file_path, null=True, blank=True) - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') def save(self, *args, **kwargs): self.software_version = settings.APP_VERSION @@ -650,7 +677,7 @@ def validate_name(alias): Alias validation. :param alias: Alias value ''' - max_length = Transient._meta.get_field('alias').max_length + max_length = Alias._meta.get_field('alias').max_length if len(alias) > max_length: raise ValidationError(f'''Invalid alias: "{alias}" is longer than the max length ''' f'''of {max_length} characters.''') @@ -659,7 +686,7 @@ def __str__(self): return (f'''"{self.alias}" is an alias for {'transient' if self.transient else 'host'} ''' f'''"{self.transient.name if self.transient else self.host.name}"''') - alias = models.CharField(max_length=64, unique=True, validators=[validate_name]) + alias = models.CharField(max_length=64, unique=True, validators=[validate_name], primary_key=True) transient = models.ForeignKey(Transient, null=True, blank=True, on_delete=models.CASCADE) host = models.ForeignKey(Host, null=True, blank=True, on_delete=models.CASCADE) objects = AliasManager() @@ -672,14 +699,16 @@ class UsageMetricsLog(models.Model): request_url (models.CharField): The requested URL request_method (models.CharField): The HTTP method of the request request_time (models.DateTimeField): Time of request. - submitted_data (models.TextField): The data submitted in the request + submitted_data (models.JSONField): The data submitted in the request + query_params (models.JSONField): The query parameters submitted in the request request_user (models.CharField): The user that made the request (if authenticated). request_ip (models.CharField): The source IP that made the request. """ request_url = models.CharField(max_length=100, blank=False) request_method = models.CharField(max_length=10, blank=False) request_time = models.DateTimeField(auto_now_add=True, blank=False) - submitted_data = models.TextField(blank=True, default='') + submitted_data = models.JSONField(blank=True, default=dict) + query_params = models.JSONField(blank=True, default=dict) request_user = models.CharField(max_length=150, blank=False) request_ip = models.CharField(max_length=45, blank=True, default='') request_user_agent = models.CharField(max_length=400, blank=True, default='') @@ -734,7 +763,8 @@ class HostSpectrum(models.Model): dec_deg = models.FloatField(null=True, blank=True) spectrum_id = models.CharField(max_length=200, null=True, blank=True) message = models.CharField(max_length=100, null=True, blank=True) - software_version = models.CharField(max_length=50, blank=True, null=True) + software_version = models.CharField(max_length=50, blank=True, null=True, + help_text='Version of Blast that generated this object') def save(self, *args, **kwargs): self.software_version = settings.APP_VERSION diff --git a/app/host/static/robots.txt b/app/host/static/robots.txt index fcb0a4e3..b14ca7a3 100644 --- a/app/host/static/robots.txt +++ b/app/host/static/robots.txt @@ -10,3 +10,4 @@ Disallow: /fetch_sed_plot/ Disallow: /fetch_host_spectrum_plot/ Disallow: /api/ Disallow: /admin/ +Disallow: /swagger-ui/ diff --git a/app/host/templates/host/information_card.html b/app/host/templates/host/information_card.html index 4f757645..59360d59 100644 --- a/app/host/templates/host/information_card.html +++ b/app/host/templates/host/information_card.html @@ -34,15 +34,25 @@ alert(`Alias is too short. Must be at least 5 characters.`) return; } - let url = `/api/alias/${newAlias}/${objectType}/${objectName}/`; + let url = `/api/alias/`; const csrfToken = getCookie('csrftoken'); let headers = new Headers({ "X-CSRFToken": csrfToken, // Note: User-Agent header is not mutable on all browsers and so // is not guaranteed to include this custom suffix. - "User-Agent": `${navigator.userAgent} web app` + "User-Agent": `${navigator.userAgent} web app`, + "Content-Type": "application/json", + "Accept": "application/json" }); - await fetch(url, { method: "POST", headers: headers }) + let data = { + "alias": newAlias, + } + if (objectType == "transient") { + data["transient"] = objectName + } else { + data["host"] = objectName + } + await fetch(url, { method: "POST", headers: headers, body: JSON.stringify(data)}) // success handling .then((result) => { console.log(JSON.stringify(result)); diff --git a/app/host/templates/host/transient_actions.html b/app/host/templates/host/transient_actions.html index a5cb4c73..e46ef1c4 100644 --- a/app/host/templates/host/transient_actions.html +++ b/app/host/templates/host/transient_actions.html @@ -10,7 +10,7 @@
Download with all files (.tar.gz)
- Download without files (.json) + Download without files (.json) {% if local_sed_results %}
diff --git a/app/host/urls.py b/app/host/urls.py index 275469e1..7d5ad69a 100644 --- a/app/host/urls.py +++ b/app/host/urls.py @@ -2,8 +2,6 @@ from django.urls import include from django.urls import path -from django.urls import re_path -from rest_framework.routers import DefaultRouter from rest_framework.schemas import get_schema_view from host.workflow import reprocess_transient_view from host.tasks import retrigger_transient_view @@ -12,7 +10,6 @@ schema_view = get_schema_view(title="Blast API") from . import views -import api.views base_path = os.environ.get("BASE_PATH", "").strip("/") if base_path != "": @@ -47,28 +44,6 @@ path(f"""{base_path}fetch_host_spectrum_plot""", views.fetch_host_spectrum_plot, name='fetch_host_spectrum_plot'), ] -router = DefaultRouter() - -router.register(r"transient", api.views.TransientViewSet) -router.register(r"aperture", api.views.ApertureViewSet) -router.register(r"cutout", api.views.CutoutViewSet, basename="cutout") -router.register(r"filter", api.views.FilterViewSet) -router.register(r"aperturephotometry", api.views.AperturePhotometryViewSet) -router.register(r"sedfittingresult", api.views.SEDFittingResultViewSet, basename="sedfittingresult") -router.register(r"taskregister", api.views.TaskRegisterViewSet) -router.register(r"task", api.views.TaskViewSet) -router.register(r"host", api.views.HostViewSet) - -# Login/Logout -api_url_patterns = [ - re_path(r"^api/", include(router.urls)), - # TODO: As of v1.5.7, this api/schema endpoint is broken: - # "AssertionError: `inflection` must be installed for OpenAPI schema support." - # re_path(r"^api/schema/$", schema_view), - re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), -] - -urlpatterns += api_url_patterns urlpatterns += [ path('api/schema/openapi/', SpectacularAPIView.as_view(), name='schema'), # Download of API Schema in YAML diff --git a/docs/requirements.txt b/docs/requirements.txt index c7e7871a..04949c07 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,6 +10,7 @@ django-silk==5.4.3 django-tables2==2.7.5 Django==5.1.14 djangorestframework==3.16.1 +drf-spectacular==0.29.0 furo==2025.9.25 git+https://github.com/jorenham/django-cron.git@cec7465#egg=django-cron # 2025/09/10 Replaced with fork due to v0.6.0 incompatibility with Django v5.1. mozilla-django-oidc==4.0.1 diff --git a/docs/usage/web_api.rst b/docs/usage/web_api.rst index 7017e15c..fccbb93b 100644 --- a/docs/usage/web_api.rst +++ b/docs/usage/web_api.rst @@ -81,8 +81,23 @@ The alias itself must be unique across both types of target objects. * :code:`GET /api/alias//` - Fetch information about an existing :code:`alias` * :code:`DELETE /api/alias//` - Delete existing :code:`alias` -* :code:`POST /api/alias//(transient|host)//` - Create a new :code:`alias` by specifying - the :code:`name` of the target object and whether the target object is a "transient" or a "host". +* :code:`POST /api/alias/` - Create a new :code:`alias` by specifying the :code:`name` of the target object and whether the target object is a "transient" or a "host". The request body should be: + + .. code-block:: json + + { + "alias": "", + "transient": "" + } + + or: + + .. code-block:: json + + { + "alias": "", + "host": "" + } .. _api_data_schema: diff --git a/run/test_api.py b/run/test_api.py deleted file mode 100644 index d655198b..00000000 --- a/run/test_api.py +++ /dev/null @@ -1,105 +0,0 @@ -import requests -import random -from requests.auth import HTTPBasicAuth -import os -import sys -import json -import datetime - -import logging -logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) -logger = logging.getLogger(__name__) - -class BlastApi(): - def __init__(self, conf): - self.conf = conf - self.conf['api_url_base'] = f'''{self.conf['api_url_protocol']}://{self.conf['api_url_authority']}''' - if self.conf['api_url_basepath']: - self.conf['api_url_base'] += f'''/{self.conf['api_url_basepath']}''' - logger.debug(self.conf) - # if self.conf['token']: - # self.api_token = self.conf['token'] - # else: - # # import base64 - # # basic_auth_credentials = base64.b64encode(bytes(f'''{self.conf['username']}:{self.conf['password']}''', 'utf-8')) - # self.api_token = self.obtain_api_token(username=self.conf['username'], password=self.conf['password']) - self.json_headers = { - # 'Authorization': f'''Token {self.api_token}''', - 'Content-Type': 'application/json', - } - def display_response(self, response, parse_json=True): - try: - assert isinstance(response, requests.Response) - # assert response.text - # assert response.status_code - except: - logger.debug(f'''Invalid response object''') - return - logger.debug(f'''Response code: {response.status_code}''') - try: - assert response.status_code in range(200,300) - if parse_json: - data = response.json() - logger.debug(json.dumps(data, indent=2)) - return data - else: - return response - except: - try: - logger.debug(f'''ERROR: {json.dumps(response.text, indent=2)}''') - except: - logger.debug(f'''ERROR: {response.text}''') - - def obtain_api_token(self, username, password): - response = requests.post( - # This URL must NOT have a trailing slash - f'''{self.conf['api_url_base']}/token''', - json={ - 'username': username, - 'password': password, - }, - headers={ - 'Content-Type': 'application/json', - }, - auth=HTTPBasicAuth(self.conf['username'], self.conf['password']), - ) - data = self.display_response(response) - return data['token'] - - def launch_workflow(self, name=f'''transient{random.randrange(10000,99999)}'''): - response = requests.put( - # The URL trailing slash is required without setting APPEND_SLASH=False - f'''{self.conf['api_url_base']}/workflow/{name}''', - headers=self.json_headers, - # auth=self.basic_auth, - ) - return self.display_response(response) - - def launch_tasks(self): - response = requests.put( - # The URL trailing slash is required without setting APPEND_SLASH=False - f'''{self.conf['api_url_base']}/launchtasks''', - headers=self.json_headers, - # auth=self.basic_auth, - ) - return self.display_response(response) - -if __name__ == "__main__": - import sys - import time - - # Import credentials and config from environment variables - api = BlastApi({ - 'username': os.environ.get('CE_USERNAME', 'admin'), - 'password': os.environ.get('CE_PASSWORD', ''), - 'token': os.environ.get('CE_API_TOKEN', ''), - 'api_url_protocol': os.environ.get('CE_API_URL_PROTOCOL', 'http'), - 'api_url_authority': os.environ.get('CE_API_URL_AUTHORITY', 'localhost:8000'), - 'api_url_basepath': os.environ.get('CE_API_URL_BASEPATH', 'api'), - }) - - transient_name = sys.argv[1] - api.launch_workflow(name=transient_name) - - # api.launch_tasks() - sys.exit()