ci: merge main to release (#8306)

Author: rjsparks

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241114T1954
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241127T2054
 LABEL maintainer="IETF Tools Team <[email protected]>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20241114T1954
+20241127T2054

dev/build/gunicorn.conf.py

@@ -12,26 +12,25 @@
             "level": "INFO",
             "handlers": ["console"],
             "propagate": False,
-            "qualname": "gunicorn.error"
+            "qualname": "gunicorn.error",
         },
-
         "gunicorn.access": {
             "level": "INFO",
             "handlers": ["access_console"],
             "propagate": False,
-            "qualname": "gunicorn.access"
-        }
+            "qualname": "gunicorn.access",
+        },
     },
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
             "formatter": "json",
-            "stream": "ext://sys.stdout"
+            "stream": "ext://sys.stdout",
         },
         "access_console": {
             "class": "logging.StreamHandler",
             "formatter": "access_json",
-            "stream": "ext://sys.stdout"
+            "stream": "ext://sys.stdout",
         },
     },
     "formatters": {
@@ -44,14 +43,29 @@
             "class": "ietf.utils.jsonlogger.GunicornRequestJsonFormatter",
             "style": "{",
             "format": "{asctime}{levelname}{message}{name}{process}",
-        }
-    }
+        },
+    },
 }
 
-def pre_request(worker, req):
+# Track in-flight requests and emit a list of what was happeningwhen a worker is terminated.
+# For the default sync worker, there will only be one request per PID, but allow for the
+# possibility of multiple requests in case we switch to a different worker class.
+#
+# This dict is only visible within a single worker, but key by pid to guarantee no conflicts.
+#
+# Use a list rather than a set to allow for the possibility of overlapping identical requests.
+in_flight_by_pid: dict[str, list[str]] = {}  # pid -> list of in-flight requests
+
+
+def _describe_request(req):
+    """Generate a consistent description of a request
+
+    The return value is used identify in-flight requests, so it must not vary between the
+    start and end of handling a request. E.g., do not include a timestamp.
+    """
     client_ip = "-"
     cf_ray = "-"
-    for (header, value) in req.headers:
+    for header, value in req.headers:
         header = header.lower()
         if header == "cf-connecting-ip":
             client_ip = value
@@ -61,4 +75,38 @@ def pre_request(worker, req):
         path = f"{req.path}?{req.query}"
     else:
         path = req.path
-    worker.log.info(f"gunicorn starting to process {req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})")
+    return f"{req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})"
+
+
+def pre_request(worker, req):
+    """Log the start of a request and add it to the in-flight list"""
+    request_description = _describe_request(req)
+    worker.log.info(f"gunicorn starting to process {request_description}")
+    in_flight = in_flight_by_pid.setdefault(worker.pid, [])
+    in_flight.append(request_description)
+
+
+def worker_abort(worker):
+    """Emit an error log if any requests were in-flight"""
+    in_flight = in_flight_by_pid.get(worker.pid, [])
+    if len(in_flight) > 0:
+        worker.log.error(
+            f"Aborted worker {worker.pid} with in-flight requests: {', '.join(in_flight)}"
+        )
+
+
+def worker_int(worker):
+    """Emit an error log if any requests were in-flight"""
+    in_flight = in_flight_by_pid.get(worker.pid, [])
+    if len(in_flight) > 0:
+        worker.log.error(
+            f"Interrupted worker {worker.pid} with in-flight requests: {', '.join(in_flight)}"
+        )
+
+
+def post_request(worker, req, environ, resp):
+    """Remove request from in-flight list when we finish handling it"""
+    request_description = _describe_request(req)
+    in_flight = in_flight_by_pid.get(worker.pid, [])
+    if request_description in in_flight:
+        in_flight.remove(request_description)

ietf/api/apps.py

@@ -12,4 +12,8 @@ def ready(self):
         interact with the database. See        
         https://docs.djangoproject.com/en/4.2/ref/applications/#django.apps.AppConfig.ready
         """
+        # Populate our API list now that the app registry is set up
         populate_api_list()
+
+        # Import drf-spectacular extensions 
+        import ietf.api.schema  # pyflakes: ignore

ietf/api/authentication.py

@@ -0,0 +1,19 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+from rest_framework import authentication
+from django.contrib.auth.models import AnonymousUser
+
+
+class ApiKeyAuthentication(authentication.BaseAuthentication):
+    """API-Key header authentication"""
+
+    def authenticate(self, request):
+        """Extract the authentication token, if present
+        
+        This does not validate the token, it just arranges for it to be available in request.auth.
+        It's up to a Permissions class to validate it for the appropriate endpoint.
+        """
+        token = request.META.get("HTTP_X_API_KEY", None)
+        if token is None:
+            return None
+        return AnonymousUser(), token  # available as request.user and request.auth

ietf/api/permissions.py

@@ -0,0 +1,39 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+from rest_framework import permissions
+from ietf.api.ietf_utils import is_valid_token
+
+
+class HasApiKey(permissions.BasePermission):
+    """Permissions class that validates a token using is_valid_token
+    
+    The view class must indicate the relevant endpoint by setting `api_key_endpoint`.
+    Must be used with an Authentication class that puts a token in request.auth.
+    """
+    def has_permission(self, request, view):
+        endpoint = getattr(view, "api_key_endpoint", None)
+        auth_token = getattr(request, "auth", None)
+        if endpoint is not None and auth_token is not None:
+            return is_valid_token(endpoint, auth_token)
+        return False
+
+
+class IsOwnPerson(permissions.BasePermission):
+    """Permission to access own Person object"""
+    def has_object_permission(self, request, view, obj):
+        if not (request.user.is_authenticated and hasattr(request.user, "person")):
+            return False
+        return obj == request.user.person
+
+
+class BelongsToOwnPerson(permissions.BasePermission):
+    """Permission to access objects associated with own Person
+    
+    Requires that the object have a "person" field that indicates ownership.
+    """
+    def has_object_permission(self, request, view, obj):
+        if not (request.user.is_authenticated and hasattr(request.user, "person")):
+            return False
+        return (
+            hasattr(obj, "person") and obj.person == request.user.person
+        )

ietf/api/routers.py

@@ -0,0 +1,16 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""Custom django-rest-framework routers"""
+from django.core.exceptions import ImproperlyConfigured
+from rest_framework import routers
+
+class PrefixedSimpleRouter(routers.SimpleRouter):
+    """SimpleRouter that adds a dot-separated prefix to its basename"""
+    def __init__(self, name_prefix="", *args, **kwargs):
+        self.name_prefix = name_prefix
+        if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".":
+            raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'")
+        super().__init__(*args, **kwargs)
+
+    def get_default_basename(self, viewset):
+        basename = super().get_default_basename(viewset)
+        return f"{self.name_prefix}.{basename}"

ietf/api/schema.py

@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+from drf_spectacular.extensions import OpenApiAuthenticationExtension
+
+
+class ApiKeyAuthenticationScheme(OpenApiAuthenticationExtension):
+    """Authentication scheme extension for the ApiKeyAuthentication
+
+    Used by drf-spectacular when rendering the OpenAPI schema
+    """
+    target_class = "ietf.api.authentication.ApiKeyAuthentication"
+    name = "apiKeyAuth"
+
+    def get_security_definition(self, auto_schema):
+        return {
+            "type": "apiKey",
+            "description": "Shared secret in the X-Api-Key header",
+            "name": "X-Api-Key",
+            "in": "header",
+        }

ietf/api/serializer.py

@@ -1,6 +1,9 @@
-# Copyright The IETF Trust 2018-2020, All Rights Reserved
+# Copyright The IETF Trust 2018-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
+"""Serialization utilities
 
+This is _not_ for django-rest-framework!
+"""
 
 import hashlib
 import json

ietf/api/tests.py

@@ -936,6 +936,8 @@ def test_api_version(self):
         r = self.client.get(url)
         data = r.json()
         self.assertEqual(data['version'], ietf.__version__+ietf.__patch__)
+        for lib in settings.ADVERTISE_VERSIONS:
+            self.assertIn(lib, data['other'])
         self.assertEqual(data['dumptime'], "2022-08-31 07:10:01 +0000")
         DumpInfo.objects.update(tz='PST8PDT')
         r = self.client.get(url)