feat: add OpenTelemetry integration (#9886)

* feat: initial otel integration

* feat: add traceparent propagation via headers and meta tag

* feat: add memcached instrumentation

* fix: add more otel service attributes

* style: Update dev/build/gunicorn.conf.py

Co-authored-by: Jennifer Richards <[email protected]>

* style: Update dev/build/gunicorn.conf.py

Co-authored-by: Jennifer Richards <[email protected]>

* style: Update dev/build/gunicorn.conf.py

Co-authored-by: Jennifer Richards <[email protected]>

* fix: Update traceparent meta tag to use otel variable

* fix: Update traceparent_id context processor to use sub-object

---------

Co-authored-by: Jennifer Richards <[email protected]>

Author: NGPixel

dev/build/gunicorn.conf.py

@@ -1,4 +1,16 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2025, All Rights Reserved
+
+import os
+import ietf
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
 
 # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto 
 # for us. Don't trust any of the other similar headers. Only trust the header if it's coming
@@ -119,3 +131,25 @@ def post_request(worker, req, environ, resp):
     in_flight = in_flight_by_pid.get(worker.pid, [])
     if request_description in in_flight:
         in_flight.remove(request_description)
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+    resource = Resource.create(attributes={
+        "service.name": "datatracker",
+        "service.version": ietf.__version__,
+        "service.instance.id": worker.pid,
+        "service.namespace": "datatracker",
+        "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev")
+    })
+
+    trace.set_tracer_provider(TracerProvider(resource=resource))
+    otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces")
+
+    trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter))
+
+    # Instrumentations
+    DjangoInstrumentor().instrument()
+    Psycopg2Instrumentor().instrument()
+    PymemcacheInstrumentor().instrument()
+    RequestsInstrumentor().instrument()

ietf/context_processors.py

@@ -5,6 +5,7 @@
 from django.conf import settings
 from django.utils import timezone
 from ietf import __version__, __patch__, __release_branch__, __release_hash__
+from opentelemetry.propagate import inject
 
 def server_mode(request):
     return {'server_mode': settings.SERVER_MODE}
@@ -51,3 +52,8 @@ def timezone_now(request):
     return {
         'timezone_now': timezone.now(),
     }
+
+def traceparent_id(request):
+    context_extras = {}
+    inject(context_extras)
+    return { "otel": context_extras }

ietf/middleware.py

@@ -8,6 +8,7 @@
 from django.http import HttpResponsePermanentRedirect
 from ietf.utils.log import log, exc_parts
 from ietf.utils.mail import log_smtp_exception
+from opentelemetry.propagate import inject
 import re
 import smtplib
 import unicodedata
@@ -99,3 +100,12 @@ def add_header(request):
         return response
 
     return add_header
+
+def add_otel_traceparent_header(get_response):
+    """Middleware to add the OpenTelemetry traceparent id header to the response"""
+    def add_header(request):
+        response = get_response(request)
+        inject(response)
+        return response
+
+    return add_header

ietf/settings.py

@@ -411,6 +411,7 @@ def skip_unreadable_post(record):
         ],
         'OPTIONS': {
             'context_processors': [
+                'ietf.context_processors.traceparent_id',
                 'django.contrib.auth.context_processors.auth',
                 'django.template.context_processors.debug',     # makes 'sql_queries' available in templates
                 'django.template.context_processors.i18n',
@@ -443,6 +444,7 @@ def skip_unreadable_post(record):
 
 
 MIDDLEWARE = [
+    "ietf.middleware.add_otel_traceparent_header",
     "django.middleware.csrf.CsrfViewMiddleware",
     "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it
     "django.middleware.common.CommonMiddleware",

ietf/templates/base.html

@@ -15,6 +15,7 @@
             {% block title %}No title{% endblock %}
         </title>
         <meta name="viewport" content="width=device-width, initial-scale=1">
+        <meta name="traceparent" content="{{ otel.traceparent }}">
         <link href="{{ settings.STATIC_IETF_ORG }}/fonts/inter/import.css" rel="stylesheet">
         <link href="{{ settings.STATIC_IETF_ORG }}/fonts/noto-sans-mono/import.css" rel="stylesheet">
         <link rel="stylesheet" href="{% static 'ietf/css/ietf.css' %}">
@@ -170,4 +171,4 @@
         </script>
       {% analytical_body_bottom %}
     </body>
-</html>
+</html>

requirements.txt

@@ -52,6 +52,12 @@ mock>=5.2.0  # should replace with unittest.mock and remove dependency
 types-mock>=5.2.0
 mypy~=1.7.0  # Version requirements determined by django-stubs.
 oic>=1.7.0  # Used only by tests
+opentelemetry-sdk>=1.38.0
+opentelemetry-instrumentation-django>=0.59b0
+opentelemetry-instrumentation-psycopg2>=0.59b0
+opentelemetry-instrumentation-pymemcache>=0.59b0
+opentelemetry-instrumentation-requests>=0.59b0
+opentelemetry-exporter-otlp-proto-http>=1.38.0
 pillow>=11.3.0
 psycopg2>=2.9.10
 pyang>=2.6.1