diff --git a/DOCKER_SETUP.md b/DOCKER_SETUP.md index 145add71..f63a11e2 100644 --- a/DOCKER_SETUP.md +++ b/DOCKER_SETUP.md @@ -87,6 +87,68 @@ Some services require credential files (JSON files for GCP, OAuth, etc.). Follow | **postgres-console** | 5433 | Console database | | **redis-floware** | 6379 | Floware cache | | **redis-call-processing** | 6380 | Call processing service cache | +| **otel-collector** | 4317/4318 | OpenTelemetry gateway (OTLP gRPC/HTTP), 13133 health, 8888 self-metrics | +| **jaeger** | 16686 | Jaeger UI for local distributed traces | + +## Observability + +Applications never talk to an APM vendor directly. They push standard OTLP to +the **otel-collector**, which fans that out along two independent pipelines: + +| Pipeline | Sampling | Processing | Destination | +|----------|----------|------------|-------------| +| `*/local_debug` | 100%, unsampled | batch only | Jaeger (`http://localhost:16686`) | +| `*/cloud_upstream` | all errors, all requests >2s, 5% of the rest | PII redaction, then tail sampling | your cloud APM | + +That split is why local debugging shows every request while the cloud bill +stays bounded, and why switching cloud vendors requires **no application +change or rebuild**. + +### View traces locally + +```bash +docker compose -f docker-compose.local.yml up -d otel-collector jaeger floware +open http://localhost:16686 +``` + +Traces carry `app.user.id`, `app.role.id`, `app.session.id` and +`app.request.id` on every span (HTTP, DB, Redis, outbound HTTP, and LLM/agent +spans), propagated via OpenTelemetry Baggage. `app.user.id` is visible in +Jaeger but hashed before it is exported to any cloud backend. + +### Push to a cloud APM backend + +Pick one overlay and set its credentials in `.env`: + +| Backend | `OTEL_EXPORTER_OVERLAY` | Credentials | +|---------|-------------------------|-------------| +| Local only (default) | `/etc/otel/exporters/none.yaml` | – | +| Any OTLP vendor (Grafana Cloud, Honeycomb, New Relic, Datadog, SigNoz, Elastic, …) | `/etc/otel/exporters/otlphttp.yaml` | `OTEL_CLOUD_ENDPOINT`, `OTEL_CLOUD_HEADERS_AUTHORIZATION` | +| Azure Application Insights | `/etc/otel/exporters/azure.yaml` | `APPLICATIONINSIGHTS_CONNECTION_STRING` | +| AWS X-Ray + CloudWatch | `/etc/otel/exporters/aws.yaml` | `AWS_REGION` + IAM role | +| GCP Cloud Trace + Monitoring | `/etc/otel/exporters/gcp.yaml` | `GOOGLE_CLOUD_PROJECT` + ADC | + +Then restart just the collector — no application rebuild: + +```bash +docker compose -f docker-compose.local.yml up -d --force-recreate otel-collector +curl -sf http://localhost:13133 && echo "collector healthy" +``` + +The overlay files live in [`otel/exporters/`](otel/exporters/); the shared +receivers, processors and local pipelines are in +[`otel/collector-base.yaml`](otel/collector-base.yaml). Adding a vendor means +adding one file there. + +> **Do not set `OTEL_TRACES_SAMPLER` to a ratio.** Head sampling must stay at +> 100% or the collector's `tail_sampling` cannot see complete traces. + +> **Scaling note:** `tail_sampling` requires every span of a trace to reach the +> same collector instance. That holds for the single container here, but +> scaling the collector past one replica needs a two-tier topology — an agent +> tier fanning out via the `loadbalancing` exporter keyed by trace ID, into a +> gateway tier that owns `tail_sampling`. Without it, sampling decisions go +> silently wrong. ## Environment Variables Reference @@ -602,7 +664,9 @@ This docker-compose setup is designed for **local development only**. For produc 1. Use Kubernetes or Docker Swarm for orchestration 2. Implement proper secrets management (Vault, AWS Secrets Manager, etc.) -3. Set up monitoring and logging (Prometheus, Grafana, ELK stack) +3. Deploy the OpenTelemetry Collector as a DaemonSet/sidecar and point + `OTEL_EXPORTER_OTLP_ENDPOINT` at it; supply cloud APM credentials from a + secret store rather than env literals (see the Observability section above) 4. Configure auto-scaling based on load 5. Use managed databases (RDS, Cloud SQL) instead of containerized databases 6. Implement backup and disaster recovery procedures diff --git a/OPENTELEMETRY_ARCHITECTURE_GUIDE.md b/OPENTELEMETRY_ARCHITECTURE_GUIDE.md new file mode 100644 index 00000000..ddb3dd2d --- /dev/null +++ b/OPENTELEMETRY_ARCHITECTURE_GUIDE.md @@ -0,0 +1,182 @@ +# Wavefront OpenTelemetry & APM Architecture: Complete Reference & Deep Dive + +This document captures the complete architectural breakdown, codebase changes, concepts, and configuration guide for the **Wavefront OpenTelemetry Observability System**. + +--- + +## 1. Executive Summary & Architecture + +Wavefront has transitioned from a legacy, pull-based Prometheus scraping model (`prometheus-client`, custom in-memory counters, and `/v1/_metrics`) to a **push-based OpenTelemetry (OTel) Distributed Tracing & APM Architecture**. + +### Core Tenet: Decoupled Multi-Cloud Export +* **Application Services** (`floware`, `celery_worker`, `call_processing`) speak only standard OpenTelemetry Protocol (**OTLP**) to a local **OpenTelemetry Collector Gateway**. +* No vendor-specific cloud SDKs (Azure Monitor, AWS X-Ray, Datadog) are imported into Python application code. +* Switching or adding cloud monitoring vendors is an **infrastructure-only configuration** (selecting a YAML overlay file in the collector) with **zero code changes and zero container rebuilds**. + +``` ++---------------------------------------------------------------------------------------------------+ +| Wavefront Application | +| | +| +--------------------------+ +--------------------------+ +---------------------------------+ | +| | floware (FastAPI) | | celery_worker | | call_processing (Voice) | | +| | - FastAPI HTTP Spans | | - flo_ai Agent Spans | | - Pipecat Voice Spans | | +| | - SQLAlchemy DB Spans | | - Background LLM Spans | | - STT / LLM / TTS Latencies | | +| | - Redis & HTTPX Spans | | - Task Duration Metrics | | - Session Metadata | | +| | - flo_ai LLM Spans | | | | | | +| +------------+-------------+ +------------+-------------+ +----------------+----------------+ | ++---------------|-----------------------------|---------------------------------|-------------------+ + | OTLP (gRPC :4317) | OTLP (gRPC :4317) | OTLP (gRPC :4317) + +-----------------------------+---------------------------------+ + | + v + +-------------------------------------------------+ + | OpenTelemetry Collector Contrib Gateway | + | (otel-collector-contrib) | + | | + | Receivers: OTLP gRPC (:4317), HTTP (:4318) | + | Processors: memory_limiter, batch, resource | + | Local Pipeline: 100% unsampled -> Jaeger | + | Cloud Pipeline: PII Redaction -> Tail Sampling | + +-----------------------+-------------------------+ + | + +------------------------+------------------------+ + | (Local Dev) | (Cloud APM - Honeycomb | (Azure / AWS / GCP / + v v Grafana Cloud, etc.) v New Relic / Datadog) + +--------------------+ +--------------------+ +--------------------+ + | Jaeger UI | | Honeycomb.io | | Azure App Insights | + | (Distributed | | (Traces, Datasets, | | AWS X-Ray/CloudW. | + | Traces UI :16686)| | BubbleUp, Latency)| | GCP Cloud Trace | + +--------------------+ +--------------------+ +--------------------+ +``` + +--- + +## 2. The 7 OpenTelemetry Python Packages Explained + +| Package | What it is | Why it was added | Where it is used in the codebase | +| :--- | :--- | :--- | :--- | +| **`opentelemetry-api`** | Abstract API contracts for traces, baggage, metrics, and context. | Allows interacting with spans and baggage without coupling code to any engine. | `baggage_middleware.py`, `errors.py`, `baggage_span_processor.py`, `bootstrap.py` | +| **`opentelemetry-sdk`** | Reference implementation of the OTel API (`TracerProvider`, `SpanProcessor`, Batch Queues). | Implements the runtime engine that generates, processes, and prepares spans for export. | `baggage_span_processor.py`, `flo_ai.telemetry` | +| **`opentelemetry-exporter-otlp`** | Protocol exporter transmitting spans/metrics over gRPC (`4317`) or HTTP (`4318`). | Enables pushing telemetry batches across the network to the `otel-collector` container. | `flo_ai.configure_telemetry()` called in `bootstrap.py` | +| **`opentelemetry-instrumentation-fastapi`** | Auto-instrumentor for FastAPI and Starlette apps. | Automatically creates root `SERVER` spans, extracts incoming W3C `traceparent` headers, records route templates, and generates HTTP duration metrics. | `bootstrap.py` (`instrument_fastapi`), called in `floware/server.py` | +| **`opentelemetry-instrumentation-sqlalchemy`** | Auto-instrumentor for SQLAlchemy engines. | Intercepts SQL queries executed on `DatabaseClient`, emitting `db.query` spans containing SQL syntax and query latencies. | `bootstrap.py` (`instrument_sqlalchemy`), called in `floware/server.py` on `db_client.engine` | +| **`opentelemetry-instrumentation-redis`** | Auto-instrumentor for `redis-py` client. | Instruments cache operations (`GET`, `SET`, pipeline commands) into child spans under the active trace. | `bootstrap.py` (`_instrument_clients`) | +| **`opentelemetry-instrumentation-httpx`** | Auto-instrumentor for async/sync `httpx` HTTP clients. | Instruments outbound HTTP calls (external APIs, webhooks, `call_processing`) and injects W3C trace context headers. | `bootstrap.py` (`_instrument_clients`) | + +--- + +## 3. Context Propagation & Baggage Architecture + +### W3C Baggage vs Span Attributes +* **Baggage:** Key-value data attached to the execution context that propagates automatically across threads, coroutines, and distributed service boundaries. +* **Problem:** In standard OpenTelemetry, baggage is transport-only and is **not** copied onto spans as searchable attributes. +* **Solution (`BaggageSpanProcessor`):** A custom `SpanProcessor` intercepts `on_start()` for every span in the application and copies `app.*` baggage keys (`app.user.id`, `app.role.id`, `app.session.id`, `app.request.id`) onto all child spans (DB queries, Redis lookups, LLM executions). + +### Raw ASGI Middleware vs `BaseHTTPMiddleware` +* Starlette's `BaseHTTPMiddleware` executes downstream handlers in a separate `anyio` task group with a **copied context**, causing `context.attach()` to fail to propagate to route handlers. +* `BaggageMiddleware` is implemented as a **raw ASGI middleware** (`async def __call__(self, scope, receive, send)`), ensuring baggage is attached directly to the active coroutine. + +### Middleware Layering Order (Outer $\rightarrow$ Inner) +FastAPI executes middleware in reverse registration order: +1. `FastAPIInstrumentor` *(Outermost — opens the `SERVER` span immediately upon request arrival)* +2. `SecurityHeadersMiddleware` *(Applies CSP and security headers)* +3. `RequireAuthMiddleware` *(Validates JWT / session and attaches `session` to `scope['state']`)* +4. `RequestIdMiddleware` *(Extracts or generates `request_id`)* +5. `BaggageMiddleware` *(Innermost — reads `session` and `request_id`, attaches baggage to context and `SERVER` span)* +6. Route Handlers & Business Logic + +### Exception Handling & Span Error Marking +When a FastAPI global exception handler catches an unhandled exception and returns a JSON 500 response without re-raising, the ASGI middleware considers the request successfully handled (`status=OK`). The helper `record_exception_on_span(exc)` in `telemetry/errors.py` explicitly records `exception.type`, `exception.message`, and `exception.stacktrace`, marking the span status as `StatusCode.ERROR`. + +--- + +## 4. OpenTelemetry Collector Architecture & Deep-Merge Overlays + +### Additive Overlay Pattern +The OpenTelemetry Collector runs with multiple `--config` flags: +```yaml +command: + - "--config=/etc/otel/collector-base.yaml" + - "--config=${OTEL_EXPORTER_OVERLAY:-/etc/otel/exporters/none.yaml}" +``` + +Because an empty exporter list (`exporters: []`) crashes the collector, `collector-base.yaml` only declares the local pipelines (`traces/local_debug`, `metrics/local_debug`). Cloud overlay files additively introduce the `traces/cloud_upstream` pipeline and specific exporter definitions. + +### Available Overlays + +| Overlay File | Destination | Required Environment Variables | +| :--- | :--- | :--- | +| [`otel/exporters/none.yaml`](otel/exporters/none.yaml) | Local Jaeger only (Default) | None | +| [`otel/exporters/honeycomb.yaml`](otel/exporters/honeycomb.yaml) | Honeycomb.io | `OTEL_CLOUD_HEADERS_AUTHORIZATION=` | +| [`otel/exporters/otlphttp.yaml`](otel/exporters/otlphttp.yaml) | Generic OTLP (Grafana Cloud, Datadog, New Relic, SigNoz) | `OTEL_CLOUD_ENDPOINT`, `OTEL_CLOUD_HEADERS_AUTHORIZATION` | +| [`otel/exporters/azure.yaml`](otel/exporters/azure.yaml) | Azure Application Insights | `APPLICATIONINSIGHTS_CONNECTION_STRING` | +| [`otel/exporters/aws.yaml`](otel/exporters/aws.yaml) | AWS X-Ray (traces) + CloudWatch EMF (metrics) | `AWS_REGION` + IAM Credentials | +| [`otel/exporters/gcp.yaml`](otel/exporters/gcp.yaml) | GCP Cloud Trace + Monitoring | `GOOGLE_CLOUD_PROJECT` + ADC | + +--- + +## 5. Tail Sampling Deep Dive + +```yaml + tail_sampling: + decision_wait: 10s + num_traces: 50000 + expected_new_traces_per_sec: 100 + policies: + - name: errors + type: status_code + status_code: { status_codes: [ERROR] } + - name: slow + type: latency + latency: { threshold_ms: 2000 } + - name: baseline-sample + type: probabilistic + probabilistic: { sampling_percentage: 5 } +``` + +### How it works: +1. **`decision_wait: 10s`**: Buffers spans for 10 seconds so asynchronous database queries, LLM token generations, and Celery tasks finish and arrive at the collector. +2. **`errors` (100% Retention)**: If *any* span in the trace encountered an error, save the entire trace. +3. **`slow` (100% Retention)**: If total trace duration was $\ge 2000\text{ ms}$, save the entire trace. +4. **`baseline-sample` (5% Retention)**: If the request was fast and successful, keep 5% to calculate baseline latency percentiles (p50/p95) while discarding 95% of routine `200 OK` traces, slashing cloud ingestion costs. + +### Scaling Past 1 Replica (Production Multi-Pod Topology): +* **Single Pod Gateway**: Handles ~1,000–3,000 spans/sec comfortably (768MB RAM limit). +* **Scaling Beyond 1 Pod**: Because `tail_sampling` requires all spans of a `trace_id` to reach the same collector instance, do not simply increase replica count behind a standard round-robin Service. Deploy a **two-tier topology**: an Agent tier (or ingress proxy) routing spans via the OpenTelemetry **`loadbalancing` exporter** (keyed by `trace_id`) into the backend Gateway tier that runs `tail_sampling`. + +--- + +## 6. Collector Self-Observability & Metrics Exposition + +```yaml + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 +``` + +* **Purpose:** Monitors the **collector container itself** (RAM usage, CPU, span queue drops, and export retry failures). +* **Format:** Exposes standard plain-text OpenMetrics/Prometheus format on `http://localhost:8888/metrics`. +* **Verification:** Run `curl http://localhost:8888/metrics` to inspect internal operational counters like `otelcol_receiver_accepted_spans` and `otelcol_exporter_sent_spans`. + +--- + +## 7. Summary of Files Changed & Created + +| File | Status | Description | +| :--- | :--- | :--- | +| [`telemetry/bootstrap.py`](wavefront/server/modules/common_module/common_module/telemetry/bootstrap.py) | **[NEW]** | Configures providers, instruments FastAPI, SQLAlchemy, Redis, and HTTPX. | +| [`telemetry/baggage_middleware.py`](wavefront/server/modules/common_module/common_module/telemetry/baggage_middleware.py) | **[NEW]** | Raw ASGI middleware injecting tenant metadata into context & `SERVER` span. | +| [`telemetry/baggage_span_processor.py`](wavefront/server/modules/common_module/common_module/telemetry/baggage_span_processor.py) | **[NEW]** | Promotes `app.*` baggage entries onto all child spans. | +| [`telemetry/errors.py`](wavefront/server/modules/common_module/common_module/telemetry/errors.py) | **[NEW]** | Attaches exception events and error status to active spans in global handlers. | +| [`otel/collector-base.yaml`](otel/collector-base.yaml) | **[NEW]** | Base OTel Collector config: receivers, processors, redaction, and local Jaeger pipeline. | +| [`otel/exporters/*.yaml`](otel/exporters/) | **[NEW]** | Additive cloud export overlays for Honeycomb, Generic OTLP, Azure, AWS, GCP, and None. | +| [`floware/server.py`](wavefront/server/apps/floware/floware/server.py) | **[MODIFY]** | Lifespan telemetry bootstrap, SQLAlchemy instrumentation, and Baggage middleware registration. | +| [`celery_app.py`](wavefront/server/background_jobs/celery_worker/celery_worker/celery_app.py) | **[MODIFY]** | Initializes OTel per worker process on `@worker_process_init.connect`. | +| [`common_module/pyproject.toml`](wavefront/server/modules/common_module/pyproject.toml) | **[MODIFY]** | Replaced `prometheus-client` with official `opentelemetry-*` packages. | +| [`prometheus_middleware.py`](wavefront/server/modules/common_module/common_module/prometheus/prometheus_middleware.py) | **[DELETED]** | Removed legacy Prometheus middleware. | +| [`docker-compose.local.yml`](docker-compose.local.yml) | **[MODIFY]** | Configured `otel-collector` (with overlay layer) and `jaeger` services. | diff --git a/README.md b/README.md index a0a8e78e..0565ea7a 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Wavefront AI is an open-source middleware platform designed to: Works seamlessly with open-source LLMs/SLMs, custom models, and proprietary AI services. - **📊 Observability, Monitoring & Evaluation** - Built-in telemetry with Grafana and Prometheus support. Track agent performance, audit trails, and guardrail enforcement in real-time. + Built-in OpenTelemetry telemetry pushed to any APM backend — local Jaeger, Azure Application Insights, AWS X-Ray/CloudWatch, GCP Cloud Trace, or any OTLP-compatible vendor. Track agent performance, audit trails, and guardrail enforcement in real-time. - **🤖 No Code Agent & Workflow Builder** Built-in capabilities to build and customize AI agents, and AI Workflows, connecting Data Sources, Knowledge Bases, in minutes diff --git a/ROADMAP.md b/ROADMAP.md index 20f8173d..c71fd4c8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -269,7 +269,7 @@ Command-line interface for configuring and managing Wavefront AI. | Feature | Description | Priority | Status | Target Release | |---------|-------------|----------|--------|----------------| | **OpenTelemetry Integration** | Full OpenTelemetry support | High | ✅ Available | v1.0.0 | -| **Prometheus Metrics** | Prometheus-compatible metrics | High | ✅ Available | v1.0.0 | +| **Pluggable Cloud APM Export** | Push traces/metrics to Azure App Insights, AWS X-Ray/CloudWatch, GCP Cloud Trace, or any OTLP vendor | High | ✅ Available | v1.0.0 | | **Grafana Dashboards** | Pre-built Grafana dashboards | High | Yet to start | v0.1.0 | | **Application Metrics** | Application-level performance metrics | High | ✅ Available | v1.0.0 | | **AI Token Tracking** | Token usage tracking per agent | High | ✅ Available | v1.0.0 | diff --git a/docker-compose.sample.yml b/docker-compose.sample.yml index 133c2e96..28b8ba39 100644 --- a/docker-compose.sample.yml +++ b/docker-compose.sample.yml @@ -81,6 +81,52 @@ services: networks: - floware-network + # OpenTelemetry Collector Contrib Gateway. Two --config files layer a + # pluggable exporter overlay on top of the local-only base config; swapping + # cloud backends is a matter of pointing OTEL_EXPORTER_OVERLAY at a different + # file in otel/exporters/ - no application code or rebuild involved. See + # otel/collector-base.yaml for the fan-out design. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.159.0 + container_name: otel-collector + restart: unless-stopped + command: + - "--config=/etc/otel/collector-base.yaml" + - "--config=${OTEL_EXPORTER_OVERLAY:-/etc/otel/exporters/none.yaml}" + mem_limit: 768m + volumes: + - ./otel:/etc/otel:ro + environment: + - APPLICATIONINSIGHTS_CONNECTION_STRING=${APPLICATIONINSIGHTS_CONNECTION_STRING:-} + - AWS_REGION=${AWS_REGION:-} + - GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT:-} + - OTEL_CLOUD_ENDPOINT=${OTEL_CLOUD_ENDPOINT:-} + - OTEL_CLOUD_HEADERS_AUTHORIZATION=${OTEL_CLOUD_HEADERS_AUTHORIZATION:-} + ports: + - "4317:4317" + - "4318:4318" + - "13133:13133" + - "8888:8888" + # No container healthcheck: the collector-contrib image is distroless (no + # shell, wget or curl) and the collector binary has no probe subcommand, so + # any in-container probe fails by construction. The health_check extension + # still serves :13133 - check it from the host with + # `curl -sf http://localhost:13133`. Dependents use service_started because + # the OTLP exporters retry, so a late collector is not a startup blocker. + networks: + - floware-network + + # Jaeger v2 for Local Distributed Tracing UI. OTLP ingest is on by default + # in v2 (no COLLECTOR_OTLP_ENABLED needed - that was a v1 all-in-one flag). + jaeger: + image: jaegertracing/jaeger:2.20.0 + container_name: jaeger + restart: unless-stopped + ports: + - "16686:16686" # Jaeger Web UI + networks: + - floware-network + # Wavefront - AI middleware web wavefront: build: @@ -221,6 +267,12 @@ services: - CALL_PROCESSING_BASE_URL= - HERMES_URL= + # ============================================ + # OpenTelemetry APM Configuration + # ============================================ + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 + - OTEL_SERVICE_NAME=wavefront-floware + # ============================================ # OAuth Configuration (Optional) # ============================================ @@ -289,6 +341,8 @@ services: condition: service_healthy redis-floware: condition: service_healthy + otel-collector: + condition: service_started networks: - floware-network @@ -377,11 +431,20 @@ services: - CALL_PROCESSING_TOKEN_PREFIX= - CALL_PROCESSING_JWT_ISSUER= + # ============================================ + # OpenTelemetry APM Configuration + # ============================================ + - CALL_PROCESSING_ENABLE_TRACING=true + - CALL_PROCESSING_OTLP_ENDPOINT=http://otel-collector:4317 + - OTEL_SERVICE_NAME=wavefront-call-processing + depends_on: redis-call-processing: condition: service_healthy floware: condition: service_started + otel-collector: + condition: service_started networks: - floware-network diff --git a/documentation/index.mdx b/documentation/index.mdx index 46e93657..d89d0f04 100644 --- a/documentation/index.mdx +++ b/documentation/index.mdx @@ -36,7 +36,7 @@ Wavefront AI is an open-source middleware platform designed to seamlessly integr Works seamlessly with open-source LLMs/SLMs, custom models, and proprietary AI services. - **📊 Observability, Monitoring & Evaluation** - Built-in telemetry with Grafana and Prometheus support. Track agent performance, audit trails, and guardrail enforcement in real-time. + Built-in OpenTelemetry telemetry pushed to any APM backend — local Jaeger, Azure Application Insights, AWS X-Ray/CloudWatch, GCP Cloud Trace, or any OTLP-compatible vendor. Track agent performance, audit trails, and guardrail enforcement in real-time. - **🤖 No Code Agent & Workflow Builder** Built-in capabilities to build and customer AI agents, and AI Workflows, connecting Data Sources, Knowledge Bases, in minutes diff --git a/otel/collector-base.yaml b/otel/collector-base.yaml new file mode 100644 index 00000000..bcab95cf --- /dev/null +++ b/otel/collector-base.yaml @@ -0,0 +1,150 @@ +# Base OpenTelemetry Collector config: local Jaeger fan-out plus the +# scaffolding a cloud pipeline needs. This file alone runs the local developer +# loop (100% unsampled traces into Jaeger) with zero further configuration. +# +# Cloud export is layered on top via a second `--config` file (see +# otel/exporters/*.yaml and docker-compose.local.yml's `command:`), never by +# editing this file or the applications. The collector deep-merges repeated +# `--config` files, but MAPS merge while LISTS are replaced wholesale — so +# every overlay restates the full `exporters` list it wants for each +# `*/cloud_upstream` pipeline, local exporters included if it wants to keep +# them. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + # Must be first in every pipeline. limit_mib (rather than limit_percentage) + # so protection doesn't silently depend on the container's memory limit + # matching the percentage math — set mem_limit on the otel-collector service + # in compose to match. + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + batch: + send_batch_size: 512 + timeout: 1s + + # Background pollers emit one root span per iteration - a redis stream read + # per consumer per second, plus connection keepalives - and outnumber real + # request traces by orders of magnitude. Matched with IsRootSpan() so the + # same commands still appear when they happen *inside* a request trace, + # where they carry actual diagnostic value. + filter/background_noise: + error_mode: ignore + traces: + span: + - 'IsRootSpan() and name == "XREADGROUP"' + - 'IsRootSpan() and name == "PING"' + + resourcedetection: + detectors: [env, system] + timeout: 2s + override: false + + # Cloud-only: strip anything that shouldn't leave the building before it + # reaches an exporter. Left out of the local pipelines on purpose, so the + # Jaeger UI still shows the raw data for debugging. + # + # NOTE: verify these OTTL function names (SHA256, replace_pattern, delete_key) + # against the pinned collector version's `transform` processor docs before + # relying on this in production; the `redaction` processor is a viable + # alternative if OTTL syntax drifts. + transform/redact: + error_mode: ignore + trace_statements: + - context: span + statements: + - delete_key(attributes, "http.request.header.authorization") + - delete_key(attributes, "http.request.header.cookie") + - set(attributes["app.user.id"], SHA256(attributes["app.user.id"])) where attributes["app.user.id"] != nil + - replace_pattern(attributes["url.full"], "\\?.*", "") + - replace_pattern(attributes["http.url"], "\\?.*", "") + - delete_key(attributes, "client.address") + - delete_key(attributes, "net.peer.ip") + - truncate_all(attributes, 256) + metric_statements: + - context: datapoint + statements: + - delete_key(attributes, "client.address") + - delete_key(attributes, "net.peer.ip") + + # Cloud-only, traces pipeline only: keep every error and every slow request, + # sample the rest. Requires every span of a trace to land on this same + # collector instance — fine for the single container this repo runs, but it + # silently makes wrong decisions the moment the collector is scaled past one + # replica. Scaling past one instance needs a two-tier topology (an agent + # tier fanning out via the `loadbalancing` exporter keyed by trace ID, into a + # gateway tier that owns tail_sampling) — out of scope here, flagged for + # whoever deploys this to production. + tail_sampling: + decision_wait: 10s + num_traces: 50000 + expected_new_traces_per_sec: 100 + policies: + - name: errors + type: status_code + status_code: { status_codes: [ERROR] } + - name: slow + type: latency + latency: { threshold_ms: 2000 } + - name: baseline-sample + type: probabilistic + probabilistic: { sampling_percentage: 5 } + +exporters: + # Local Distributed Tracing UI (Jaeger v2, OTLP-native). + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + + debug: + verbosity: basic + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + + pipelines: + # 100% of traces, no sampling, straight to Jaeger. Fast feedback loop for + # local development; this pipeline is not present in any cloud overlay. + traces/local_debug: + receivers: [otlp] + processors: [memory_limiter, filter/background_noise, resourcedetection, batch] + exporters: [otlp/jaeger, debug] + + metrics/local_debug: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, batch] + exporters: [debug] + + # No `*/cloud_upstream` pipeline is defined here on purpose. A collector + # pipeline must have at least one exporter, so a pipeline with an empty + # `exporters: []` fails validation at startup — there is no config-only way + # to define a pipeline as "off". Instead, each file in otel/exporters/ that + # actually wants cloud export ADDS the `traces/cloud_upstream` and + # `metrics/cloud_upstream` pipeline keys itself (service.pipelines merges + # as a map, so new keys layer in cleanly), referencing the transform/redact + # and tail_sampling processors already defined above. otel/exporters/none.yaml + # adds nothing, so with no overlay selected only the two local_debug + # pipelines above exist and the collector runs local-only. diff --git a/otel/exporters/aws.yaml b/otel/exporters/aws.yaml new file mode 100644 index 00000000..2d00ba4e --- /dev/null +++ b/otel/exporters/aws.yaml @@ -0,0 +1,33 @@ +# AWS X-Ray + CloudWatch overlay. Neither ingests raw OTLP, so this uses the +# native `awsxray` (traces) and `awsemf` (metrics) exporters. +# +# Select it with: +# OTEL_EXPORTER_OVERLAY=/etc/otel/exporters/aws.yaml +# AWS_REGION=us-east-1 +# +# Auth is IAM, not an API key: the collector container's role/credentials need +# AWSXRayDaemonWriteAccess (traces) and CloudWatchAgentServerPolicy (metrics). +# In this repo's local compose that's whatever AWS_* env vars are already +# passed through; in a real deployment it should be an instance/task role +# rather than long-lived keys. + +exporters: + awsxray: + region: ${env:AWS_REGION} + + awsemf: + region: ${env:AWS_REGION} + namespace: Wavefront + log_group_name: '/wavefront/otel-metrics' + +service: + pipelines: + traces/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, filter/background_noise, resourcedetection, transform/redact, tail_sampling, batch] + exporters: [awsxray] + + metrics/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/redact, batch] + exporters: [awsemf] diff --git a/otel/exporters/azure.yaml b/otel/exporters/azure.yaml new file mode 100644 index 00000000..1ff75475 --- /dev/null +++ b/otel/exporters/azure.yaml @@ -0,0 +1,25 @@ +# Azure Application Insights overlay. App Insights does not ingest raw OTLP, +# so this uses the native `azuremonitor` exporter rather than otlphttp. +# +# Select it with: +# OTEL_EXPORTER_OVERLAY=/etc/otel/exporters/azure.yaml +# APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=...;IngestionEndpoint=... +# +# Requires the `otelcontribcol` build to include the azuremonitorexporter +# component (the standard otel/opentelemetry-collector-contrib image does). + +exporters: + azuremonitor: + connection_string: ${env:APPLICATIONINSIGHTS_CONNECTION_STRING} + +service: + pipelines: + traces/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, filter/background_noise, resourcedetection, transform/redact, tail_sampling, batch] + exporters: [azuremonitor] + + metrics/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/redact, batch] + exporters: [azuremonitor] diff --git a/otel/exporters/gcp.yaml b/otel/exporters/gcp.yaml new file mode 100644 index 00000000..b638e7a8 --- /dev/null +++ b/otel/exporters/gcp.yaml @@ -0,0 +1,26 @@ +# GCP Cloud Trace + Cloud Monitoring overlay, for symmetry with the GAR/GCP +# build targets already used in .github/workflows/. +# +# Select it with: +# OTEL_EXPORTER_OVERLAY=/etc/otel/exporters/gcp.yaml +# GOOGLE_CLOUD_PROJECT= +# +# Auth is Application Default Credentials — mount a service account key or run +# on GCP infra with the collector's runtime identity granted +# roles/cloudtrace.agent and roles/monitoring.metricWriter. + +exporters: + googlecloud: + project: ${env:GOOGLE_CLOUD_PROJECT} + +service: + pipelines: + traces/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, filter/background_noise, resourcedetection, transform/redact, tail_sampling, batch] + exporters: [googlecloud] + + metrics/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/redact, batch] + exporters: [googlecloud] diff --git a/otel/exporters/honeycomb.yaml b/otel/exporters/honeycomb.yaml new file mode 100644 index 00000000..79ade914 --- /dev/null +++ b/otel/exporters/honeycomb.yaml @@ -0,0 +1,25 @@ +# Honeycomb.io overlay +# +# Select it with: +# OTEL_EXPORTER_OVERLAY=/etc/otel/exporters/honeycomb.yaml +# OTEL_CLOUD_HEADERS_AUTHORIZATION= +# +# (Optionally set OTEL_CLOUD_ENDPOINT if using Honeycomb EU: https://api.eu1.honeycomb.io) + +exporters: + otlphttp/cloud: + endpoint: ${env:OTEL_CLOUD_ENDPOINT} + headers: + x-honeycomb-team: ${env:OTEL_CLOUD_HEADERS_AUTHORIZATION} + +service: + pipelines: + traces/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, filter/background_noise, resourcedetection, transform/redact, tail_sampling, batch] + exporters: [otlphttp/cloud] + + metrics/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/redact, batch] + exporters: [otlphttp/cloud] diff --git a/otel/exporters/none.yaml b/otel/exporters/none.yaml new file mode 100644 index 00000000..2bb99a91 --- /dev/null +++ b/otel/exporters/none.yaml @@ -0,0 +1,4 @@ +# Default overlay: no cloud export. Adds nothing to collector-base.yaml, so +# only the local Jaeger pipelines run. This is what docker-compose.local.yml +# and docker-compose.sample.yml use unless OTEL_EXPORTER_OVERLAY is set. +{} diff --git a/otel/exporters/otlphttp.yaml b/otel/exporters/otlphttp.yaml new file mode 100644 index 00000000..b5675c15 --- /dev/null +++ b/otel/exporters/otlphttp.yaml @@ -0,0 +1,39 @@ +# Generic OTLP-over-HTTP overlay: the "plug in any cloud service" path. +# +# Covers essentially any backend that ingests raw OTLP over HTTP with a bearer +# or API-key header — Grafana Cloud / Tempo / Mimir, Honeycomb, New Relic, +# Datadog's OTLP intake, SigNoz, Elastic, Dynatrace, Axiom, Coralogix, +# Uptrace, and self-hosted collectors. If what you have is an API key rather +# than a native cloud SDK integration, use this file. +# +# Select it with: +# OTEL_EXPORTER_OVERLAY=/etc/otel/exporters/otlphttp.yaml +# and set OTEL_CLOUD_ENDPOINT / OTEL_CLOUD_HEADERS_AUTHORIZATION accordingly, +# e.g. for Grafana Cloud: +# OTEL_CLOUD_ENDPOINT=https://otlp-gateway-.grafana.net/otlp +# OTEL_CLOUD_HEADERS_AUTHORIZATION=Basic +# or for Honeycomb: +# OTEL_CLOUD_ENDPOINT=https://api.honeycomb.io +# OTEL_CLOUD_HEADERS_AUTHORIZATION= # sent as x-honeycomb-team below; +# # adjust the header key per vendor. +# +# ${env:VAR} is required syntax on the pinned collector version — a bare +# ${VAR} is not resolved. + +exporters: + otlphttp/cloud: + endpoint: ${env:OTEL_CLOUD_ENDPOINT} + headers: + Authorization: ${env:OTEL_CLOUD_HEADERS_AUTHORIZATION} + +service: + pipelines: + traces/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, filter/background_noise, resourcedetection, transform/redact, tail_sampling, batch] + exporters: [otlphttp/cloud] + + metrics/cloud_upstream: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/redact, batch] + exporters: [otlphttp/cloud] diff --git a/wavefront/server/apps/floware/floware/server.py b/wavefront/server/apps/floware/floware/server.py index d72130de..42f0c6d5 100644 --- a/wavefront/server/apps/floware/floware/server.py +++ b/wavefront/server/apps/floware/floware/server.py @@ -24,7 +24,14 @@ get_current_request_id, ) from common_module.log.logger import logger -from common_module.prometheus.prometheus_middleware import PrometheusMiddleware +from common_module.telemetry import ( + BaggageMiddleware, + configure_telemetry_providers, + instrument_fastapi, + instrument_sqlalchemy, + record_exception_on_span, + shutdown_telemetry, +) from common_module.response_formatter import ResponseFormatter from db_repo_module.cache.azure_redis_auth import patch_redis_for_azure from db_repo_module.database.connection import DatabaseClient @@ -245,6 +252,9 @@ async def lifespan(app: FastAPI): logger.info('========== Establishing db connection ...') await db_client.connect() logger.info('========== DB connection established.') + # Emits db.* spans for every query; needs the engine, so it can + # only happen once the DI container has built the client. + instrument_sqlalchemy(db_client.engine) else: raise TypeError('db_client is not an instance of DatabaseClient') @@ -338,11 +348,19 @@ def _run_trigger_renewer_sync() -> None: except Exception as e: logger.error(f'Error during application lifecycle: {str(e)}') raise + finally: + # In a `finally` so buffered spans and metrics are still flushed when + # shutdown takes an error path or startup fails before `yield`. + shutdown_telemetry() # Define FastAPI app with the lifespan context manager app = FastAPI(lifespan=lifespan) +# Providers must exist before any instrumentation is attached. The FastAPI app +# itself is instrumented further down, after all other middleware is registered. +configure_telemetry_providers(default_service_name='wavefront-floware') + floware_base_url = os.getenv('FLOWARE_BASE_URL', 'http://localhost:8001') @@ -388,18 +406,16 @@ def custom_openapi() -> dict[str, Any]: app.openapi = cast(OpenApiCallable, custom_openapi) # type: ignore[assignment] -@app.get('/v1/_metrics') -async def metrics(request: Request): - logger.debug('Metrics endpoint called') - metrics_data = await PrometheusMiddleware.metrics_endpoint(request) - return metrics_data - - # Add middleware setup +# Starlette makes the *last* `add_middleware` call the outermost layer, so +# execution order (outer -> inner) here is: +# CORS -> SecurityHeaders -> RequireAuth -> RequestId -> Baggage -> router +# BaggageMiddleware is added first (innermost) so it runs after RequireAuth has +# set request.state.session and RequestId has set the request-id context var. +app.add_middleware(_middleware(BaggageMiddleware)) app.add_middleware(_middleware(RequestIdMiddleware)) app.add_middleware(_middleware(RequireAuthMiddleware)) -app.add_middleware(_middleware(PrometheusMiddleware)) app.add_middleware(_middleware(SecurityHeadersMiddleware)) # disable to see swaggerUI origins = os.getenv('ALLOWED_ORIGINS', 'http://localhost:5173') @@ -425,6 +441,11 @@ async def metrics(request: Request): ], ) +# Instrumenting last makes the OTel ASGI middleware the outermost layer, so the +# SERVER span wraps CORS, security headers and auth rather than starting after +# them. This is the sole source of HTTP spans and HTTP metrics for this app. +instrument_fastapi(app) + # Include routers app.include_router(notification_router, prefix='/floware') app.include_router(user_management_router, prefix='/floware') @@ -469,10 +490,10 @@ async def global_exception_handler(request: Request, exc: Exception): if isinstance(exc, HTTPException): raise exc - prometheus_middleware = PrometheusMiddleware.get_instance() - if prometheus_middleware: - labels = prometheus_middleware.get_labels(request) - prometheus_middleware.http_errors_total.labels(**labels, status_code=500).inc() + # This handler swallows the exception and returns a 500 without re-raising, + # so it never reaches the OTel ASGI middleware's own exception recording - + # the SERVER span would otherwise be marked as a success. Record it here. + record_exception_on_span(exc) error_message = 'An unexpected error has occurred while performing this action, please try again' error_message += f' - {str(exc)}' diff --git a/wavefront/server/background_jobs/celery_worker/celery_worker/celery_app.py b/wavefront/server/background_jobs/celery_worker/celery_worker/celery_app.py index 81cf17f5..f584eee4 100644 --- a/wavefront/server/background_jobs/celery_worker/celery_worker/celery_app.py +++ b/wavefront/server/background_jobs/celery_worker/celery_worker/celery_app.py @@ -15,12 +15,32 @@ def setup_azure_redis_auth(**kwargs): patch_redis_for_azure() +@worker_process_init.connect +def setup_telemetry(**kwargs): + """Configure OpenTelemetry once per worker process, before any task runs. + + Previously this lived inside `get_services()` and only ran when the first + task built services, so anything before that — and any code path that + doesn't go through `get_services()` — was untraced. Doing it here, at + process start, covers the whole worker lifetime. + """ + from common_module.telemetry import configure_telemetry_providers + + configure_telemetry_providers(default_service_name='wavefront-celery-worker') + + def teardown_event_loop(**kwargs): from celery_worker.worker_setup import close_event_loop close_event_loop() +def teardown_telemetry(**kwargs): + from common_module.telemetry import shutdown_telemetry + + shutdown_telemetry() + + # Only the prefork pool dispatches worker_process_shutdown (celery/concurrency/ # prefork.py) — under solo it never fires, and solo is where the loop lives in # the main process. Hook the worker-level signal too, which fires for every @@ -28,6 +48,13 @@ def teardown_event_loop(**kwargs): worker_process_shutdown.connect(teardown_event_loop, weak=False) worker_shutdown.connect(teardown_event_loop, weak=False) +# Same prefork-vs-solo split as above, connected after the event-loop teardown +# so any pending async work has already wound down. shutdown_telemetry() is +# safe to call unconditionally (a no-op if telemetry was never configured), so +# a double-fire across both signals is harmless. +worker_process_shutdown.connect(teardown_telemetry, weak=False) +worker_shutdown.connect(teardown_telemetry, weak=False) + app = Celery('async_executor') app.conf.update( diff --git a/wavefront/server/background_jobs/celery_worker/celery_worker/worker_setup.py b/wavefront/server/background_jobs/celery_worker/celery_worker/worker_setup.py index 83cc4a89..e0cc8caf 100644 --- a/wavefront/server/background_jobs/celery_worker/celery_worker/worker_setup.py +++ b/wavefront/server/background_jobs/celery_worker/celery_worker/worker_setup.py @@ -161,6 +161,11 @@ def close_event_loop() -> None: except Exception as exc: logger.warning(f'Error draining worker event loop on shutdown: {exc}') finally: + # Telemetry shutdown is handled by the `worker_shutdown` / + # `worker_process_shutdown` signals in celery_app.py, not here — + # this function's contract is event-loop teardown, and coupling + # span/metric flush to it meant a repeated or skipped call here + # could lose the tail of the traces. _loop.close() _loop = None diff --git a/wavefront/server/modules/common_module/common_module/prometheus/prometheus_middleware.py b/wavefront/server/modules/common_module/common_module/prometheus/prometheus_middleware.py deleted file mode 100644 index c01ad77b..00000000 --- a/wavefront/server/modules/common_module/common_module/prometheus/prometheus_middleware.py +++ /dev/null @@ -1,120 +0,0 @@ -import time -from typing import Callable, Optional - -from fastapi import Request -from fastapi import Response -from prometheus_client import Counter -from prometheus_client import Gauge -from prometheus_client import Histogram -from prometheus_client import REGISTRY -from prometheus_client.openmetrics.exposition import generate_latest -from starlette.middleware.base import BaseHTTPMiddleware - - -class PrometheusMiddleware(BaseHTTPMiddleware): - _instance: Optional['PrometheusMiddleware'] = None - - def __init__(self, app): - super().__init__(app) - PrometheusMiddleware._instance = self - - # Common labels that will be used across all metrics - self.common_labels = ['module', 'instance'] - - # HTTP metrics - self.http_requests_total = Counter( - 'http_requests_total', - 'Total number of HTTP requests', - self.common_labels + ['method', 'endpoint', 'status_code'], - ) - - self.http_request_duration = Histogram( - 'http_request_duration_seconds', - 'HTTP request duration in seconds', - self.common_labels + ['method', 'endpoint'], - ) - - self.http_requests_in_progress = Gauge( - 'http_requests_in_progress', - 'Number of HTTP requests in progress', - self.common_labels + ['method', 'endpoint'], - ) - - self.http_errors_total = Counter( - 'http_errors_total', - 'Total number of HTTP errors', - self.common_labels + ['method', 'endpoint', 'status_code'], - ) - - @classmethod - def get_instance(cls) -> Optional['PrometheusMiddleware']: - """Get the singleton instance of PrometheusMiddleware""" - return cls._instance - - def get_labels(self, request: Request) -> dict: - """Extract common labels from request""" - return { - 'module': request.url.path.split('/')[3] - if len(request.url.path.split('/')) > 3 - else 'root', - 'instance': f'{request.client.host}:{request.url.port}' - if request.client - else 'unknown:unknown', - 'method': request.method, - 'endpoint': request.url.path, - } - - async def dispatch(self, request: Request, call_next: Callable) -> Response: - # Skip metrics endpoint to avoid infinite recursion - if request.url.path == '/v1/_metrics': - return await call_next(request) - - # Get common labels - labels = self.get_labels(request) - - # Record request start - self.http_requests_in_progress.labels(**labels).inc() - - # Start timing - start_time = time.time() - - try: - # Process the request - response = await call_next(request) - - # Track errors for 4xx and 5xx status codes - ADD THIS - if response and response.status_code >= 400: - self.http_errors_total.labels( - **labels, status_code=response.status_code - ).inc() - - # Record request duration - duration = time.time() - start_time - self.http_request_duration.labels(**labels).observe(duration) - - # Record request completion - self.http_requests_total.labels( - **labels, status_code=response.status_code - ).inc() - - return response - - except Exception as e: - # Record error - self.http_requests_total.labels( - **labels, status_code=getattr(e, 'status_code', 500) - ).inc() - - self.http_errors_total.labels( - **labels, status_code=getattr(e, 'status_code', 500) - ).inc() - - raise - finally: - # Decrement in-progress counter - self.http_requests_in_progress.labels(**labels).dec() - - @staticmethod - async def metrics_endpoint(request: Request) -> Response: - """Endpoint to expose Prometheus metrics""" - return Response(content=generate_latest(REGISTRY), media_type='text/plain') diff --git a/wavefront/server/modules/common_module/common_module/telemetry/__init__.py b/wavefront/server/modules/common_module/common_module/telemetry/__init__.py new file mode 100644 index 00000000..50461851 --- /dev/null +++ b/wavefront/server/modules/common_module/common_module/telemetry/__init__.py @@ -0,0 +1,19 @@ +from .baggage_middleware import BaggageMiddleware +from .baggage_span_processor import BaggageSpanProcessor +from .bootstrap import configure_telemetry_providers +from .bootstrap import instrument_fastapi +from .bootstrap import instrument_sqlalchemy +from .bootstrap import shutdown_telemetry +from .bootstrap import telemetry_endpoint +from .errors import record_exception_on_span + +__all__ = [ + 'BaggageMiddleware', + 'BaggageSpanProcessor', + 'configure_telemetry_providers', + 'instrument_fastapi', + 'instrument_sqlalchemy', + 'record_exception_on_span', + 'shutdown_telemetry', + 'telemetry_endpoint', +] diff --git a/wavefront/server/modules/common_module/common_module/telemetry/baggage_middleware.py b/wavefront/server/modules/common_module/common_module/telemetry/baggage_middleware.py new file mode 100644 index 00000000..41fe1275 --- /dev/null +++ b/wavefront/server/modules/common_module/common_module/telemetry/baggage_middleware.py @@ -0,0 +1,94 @@ +from typing import Any, Dict + +from opentelemetry import baggage +from opentelemetry import context +from opentelemetry import trace + +from common_module.middleware.request_id_middleware import get_current_request_id + +# Custom domain attributes use the `app.` prefix so they never collide with +# OpenTelemetry semantic conventions. +USER_ID_KEY = 'app.user.id' +ROLE_ID_KEY = 'app.role.id' +SESSION_ID_KEY = 'app.session.id' +REQUEST_ID_KEY = 'app.request.id' + +_SESSION_FIELDS = ( + ('user_id', USER_ID_KEY), + ('role_id', ROLE_ID_KEY), + ('session_id', SESSION_ID_KEY), +) + + +class BaggageMiddleware: + """Puts multi-tenant / business context into OpenTelemetry Baggage. + + Values are read from the authenticated ``UserSession`` that + ``RequireAuthMiddleware`` places on the request, plus the request ID, so no + caller has to send additional headers for this to work. + + This is deliberately raw ASGI middleware rather than ``BaseHTTPMiddleware``: + the latter runs the downstream app in a separate anyio task with a *copied* + context, so a ``context.attach()`` performed inside it does not reliably + reach the route handlers. A raw ASGI middleware attaches on the same + coroutine, so the baggage is visible to every span created downstream. + + Must be registered so it runs *inside* ``RequireAuthMiddleware`` and + ``RequestIdMiddleware`` (i.e. added to the app before them), and *inside* + the FastAPI instrumentation, so the SERVER span is already open and can be + annotated directly. + + Note that ``app.user.id`` is carried in the clear here on purpose: the local + Jaeger pipeline keeps it for debugging, and the collector hashes it before + anything is exported to a cloud backend. + """ + + def __init__(self, app: Any) -> None: + self.app = app + + async def __call__(self, scope: Dict[str, Any], receive: Any, send: Any) -> None: + if scope.get('type') != 'http': + await self.app(scope, receive, send) + return + + entries = self._collect(scope) + if not entries: + await self.app(scope, receive, send) + return + + ctx = context.get_current() + for key, value in entries.items(): + ctx = baggage.set_baggage(key, value, context=ctx) + + # The SERVER span is already open (the OTel ASGI middleware sits further + # out), so it predates this baggage and BaggageSpanProcessor cannot see + # it. Annotate it directly; child spans inherit via the processor. + span = trace.get_current_span() + if span.is_recording(): + for key, value in entries.items(): + span.set_attribute(key, value) + + token = context.attach(ctx) + try: + await self.app(scope, receive, send) + finally: + context.detach(token) + + @staticmethod + def _collect(scope: Dict[str, Any]) -> Dict[str, str]: + entries: Dict[str, str] = {} + + # `request.state` is backed by `scope['state']`, so the session that + # RequireAuthMiddleware assigned is readable here without a Request. + session = (scope.get('state') or {}).get('session') + if session is not None: + for attribute, key in _SESSION_FIELDS: + value = getattr(session, attribute, None) + if value: + entries[key] = str(value) + + request_id = get_current_request_id() + if request_id and request_id != 'NO-REQUEST-ID': + entries[REQUEST_ID_KEY] = request_id + + return entries diff --git a/wavefront/server/modules/common_module/common_module/telemetry/baggage_span_processor.py b/wavefront/server/modules/common_module/common_module/telemetry/baggage_span_processor.py new file mode 100644 index 00000000..8e1b08f4 --- /dev/null +++ b/wavefront/server/modules/common_module/common_module/telemetry/baggage_span_processor.py @@ -0,0 +1,37 @@ +from typing import Optional + +from opentelemetry import baggage +from opentelemetry.context import Context +from opentelemetry.sdk.trace import Span +from opentelemetry.sdk.trace import SpanProcessor + + +class BaggageSpanProcessor(SpanProcessor): + """Promotes ``app.*`` baggage entries onto every span as attributes. + + OpenTelemetry baggage travels on the context and the ``baggage`` header but + is never recorded on spans automatically. This processor copies the business + context established by ``BaggageMiddleware`` onto each span as it starts, so + DB, cache, HTTP-client and LLM child spans all carry it without any call + site having to thread it through manually. + + Only the ``app.`` prefix is promoted, which keeps arbitrary inbound baggage + from third parties out of our telemetry. + """ + + def __init__(self, prefix: str = 'app.') -> None: + self._prefix = prefix + + def on_start(self, span: Span, parent_context: Optional[Context] = None) -> None: + for key, value in baggage.get_all(context=parent_context).items(): + if key.startswith(self._prefix) and value is not None: + span.set_attribute(key, str(value)) + + def on_end(self, span: Span) -> None: + return None + + def shutdown(self) -> None: + return None + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True diff --git a/wavefront/server/modules/common_module/common_module/telemetry/bootstrap.py b/wavefront/server/modules/common_module/common_module/telemetry/bootstrap.py new file mode 100644 index 00000000..0d64822f --- /dev/null +++ b/wavefront/server/modules/common_module/common_module/telemetry/bootstrap.py @@ -0,0 +1,239 @@ +"""Vendor-agnostic OpenTelemetry bootstrap for Wavefront services. + +Only ``opentelemetry-api`` / ``opentelemetry-sdk`` and the official +``opentelemetry-instrumentation-*`` packages are used here. No cloud vendor SDK +(Azure Monitor, AWS X-Ray, Datadog, ...) is ever imported into application code: +fan-out, sampling, PII redaction and backend authentication all live in the +OpenTelemetry Collector. Switching APM backends is therefore a collector +configuration change with no application rebuild. + +Every service speaks plain OTLP to the collector and nothing else. +""" + +import os +import socket +from typing import Any, Dict, Optional + +from opentelemetry import trace + +from common_module.log.logger import logger +from common_module.telemetry.baggage_span_processor import BaggageSpanProcessor + +# Endpoints that generate telemetry noise without diagnostic value. Matched as +# regexes against the request URL by the FastAPI instrumentation. +EXCLUDED_URLS = 'health,healthz,docs,openapi.json,redoc,favicon.ico' + +_providers_configured = False +_sqlalchemy_instrumented = False + + +def telemetry_endpoint() -> Optional[str]: + """Return the collector endpoint, or ``None`` when telemetry is disabled.""" + return os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT') or None + + +def _service_name(default: str) -> str: + return os.getenv('OTEL_SERVICE_NAME') or os.getenv('APP_NAME') or default + + +def _resource_attributes() -> Dict[str, Any]: + """Extra resource attributes beyond service name/version/environment. + + ``service.instance.id`` lets a backend tell replicas apart. It replaces the + per-request client-IP ``instance`` label the old Prometheus middleware used, + which was both unbounded in cardinality and personally identifying. + """ + instance_id = os.getenv('HOSTNAME') or socket.gethostname() + return { + 'service.instance.id': f'{instance_id}:{os.getpid()}', + } + + +def _rebuild_flo_ai_metric_singletons() -> None: + """Re-create flo_ai's metric holders now that a MeterProvider exists. + + ``flo_ai.telemetry.instrumentation`` builds ``llm_metrics``, ``agent_metrics`` + and ``workflow_metrics`` at import time, and each constructor calls + ``get_meter()`` — which returns ``None`` until ``configure_telemetry()`` has + run. Every ``record_*`` call then short-circuits, so LLM/agent/workflow + metrics are silently never emitted. + + Import ordering cannot avoid this: ``flo_ai/__init__.py`` pulls the + instrumentation module in transitively, so those constructors have already + run before any ``configure_telemetry`` call is reachable. Rebuilding the + module-level singletons afterwards works because the decorators resolve them + as module globals at call time. + + The real fix is lazy meter resolution inside flo_ai; remove this once a + flo-ai release carries it. + """ + try: + import flo_ai.telemetry.instrumentation as instrumentation + + instrumentation.llm_metrics = instrumentation.LLMMetrics() + instrumentation.agent_metrics = instrumentation.AgentMetrics() + instrumentation.workflow_metrics = instrumentation.WorkflowMetrics() + except Exception as exc: + logger.error( + f'Could not rebuild flo_ai metric instruments; LLM/agent metrics ' + f'will not be emitted: {exc}', + exc_info=True, + ) + + +def configure_telemetry_providers(default_service_name: str) -> bool: + """Set up trace/metric providers and library instrumentation. + + Returns ``True`` when telemetry was configured, ``False`` when it is + disabled because no collector endpoint is set. Never raises: a broken + telemetry setup must not stop a service from serving traffic. + """ + global _providers_configured + + if _providers_configured: + return True + + otlp_endpoint = telemetry_endpoint() + if not otlp_endpoint: + logger.info('OTEL_EXPORTER_OTLP_ENDPOINT is not set; OpenTelemetry is disabled') + return False + + service_name = _service_name(default_service_name) + + try: + from flo_ai import configure_telemetry + + configure_telemetry( + service_name=service_name, + service_version=os.getenv('APP_VERSION', '0.1.0'), + environment=os.getenv('APP_ENV', 'dev'), + otlp_endpoint=otlp_endpoint, + additional_attributes=_resource_attributes(), + ) + _rebuild_flo_ai_metric_singletons() + + # flo_ai installs its TracerProvider as the global one, so the baggage + # processor can be attached to it after the fact. + tracer_provider = trace.get_tracer_provider() + if hasattr(tracer_provider, 'add_span_processor'): + tracer_provider.add_span_processor(BaggageSpanProcessor()) + + _instrument_clients() + + _providers_configured = True + logger.info( + f'OpenTelemetry configured for service "{service_name}" ' + f'(env={os.getenv("APP_ENV", "dev")}) exporting to {otlp_endpoint}' + ) + return True + except Exception as exc: + logger.error(f'Failed to initialize OpenTelemetry: {exc}', exc_info=True) + return False + + +def _instrument_clients() -> None: + """Instrument outbound clients so their spans join the request trace.""" + try: + from opentelemetry.instrumentation.redis import RedisInstrumentor + + RedisInstrumentor().instrument() + except Exception as exc: + logger.warning(f'Redis instrumentation unavailable: {exc}') + + try: + from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor + + HTTPXClientInstrumentor().instrument() + except Exception as exc: + logger.warning(f'HTTPX instrumentation unavailable: {exc}') + + +def instrument_fastapi(app: Any) -> None: + """Attach the OpenTelemetry ASGI middleware to a FastAPI app. + + Call this *after* every other ``add_middleware`` call. Starlette inserts + middleware at position 0, so the last one registered is the outermost — and + the SERVER span should wrap auth, security headers and CORS rather than + starting inside them. + + This is the only source of HTTP spans and HTTP metrics. It emits standard + semantic conventions and route-template span names, and extracts inbound + trace context from request headers, so no hand-written HTTP middleware is + needed or wanted alongside it. + + The "already done" check is per app object, never a module-level flag. + ``python server.py`` executes the server module up to three times in one + process tree - as ``__main__``, as ``__mp_main__`` in uvicorn's spawned + reload/worker child, and again as ``server`` when uvicorn imports the + ``"server:app"`` string - and each execution builds a *different* FastAPI + instance. Only the last one is served. A process-global flag is set by the + first instance and silently skips the one that actually handles requests, + which costs every HTTP SERVER span while leaving the redis/httpx/SQLAlchemy + spans (global monkey-patches, not per-app) working - so telemetry looks + half-alive rather than broken. + """ + if not _providers_configured: + return + + # Set by FastAPIInstrumentor itself; instrumenting twice only warns, but + # checking keeps the log honest. + if getattr(app, '_is_instrumented_by_opentelemetry', False): + return + + try: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + # `exclude_spans` drops the per-ASGI-event `http send` / `http receive` + # INTERNAL spans. They restate timings the SERVER span already carries, + # but there are three or more of them per request - the majority of + # every request trace, and the part that grows fastest with traffic. + FastAPIInstrumentor.instrument_app( + app, + excluded_urls=EXCLUDED_URLS, + exclude_spans=['receive', 'send'], + ) + logger.info(f'FastAPI instrumentation enabled (app id={id(app):#x})') + except Exception as exc: + logger.error(f'Failed to instrument FastAPI app: {exc}', exc_info=True) + + +def instrument_sqlalchemy(engine: Any) -> None: + """Instrument a SQLAlchemy engine so queries appear as ``db.*`` spans. + + ``engine`` is expected to be an ``AsyncEngine``. The instrumentation must be + given the underlying *sync* engine — handing it an ``AsyncEngine`` directly + silently produces no spans. + """ + global _sqlalchemy_instrumented + + if _sqlalchemy_instrumented or not _providers_configured or engine is None: + return + + try: + from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor + + SQLAlchemyInstrumentor().instrument( + engine=getattr(engine, 'sync_engine', engine) + ) + _sqlalchemy_instrumented = True + logger.info('SQLAlchemy instrumentation enabled') + except Exception as exc: + logger.warning(f'SQLAlchemy instrumentation unavailable: {exc}') + + +def shutdown_telemetry() -> None: + """Flush and shut down the telemetry providers. + + Safe to call unconditionally; call it from a ``finally`` so buffered spans + and metrics are not lost when shutdown takes an error path. + """ + if not _providers_configured: + return + + try: + from flo_ai import shutdown_telemetry as flo_shutdown + + flo_shutdown() + logger.info('OpenTelemetry providers shut down') + except Exception as exc: + logger.warning(f'Error shutting down OpenTelemetry: {exc}') diff --git a/wavefront/server/modules/common_module/common_module/telemetry/errors.py b/wavefront/server/modules/common_module/common_module/telemetry/errors.py new file mode 100644 index 00000000..de9e082b --- /dev/null +++ b/wavefront/server/modules/common_module/common_module/telemetry/errors.py @@ -0,0 +1,23 @@ +from opentelemetry import trace +from opentelemetry.trace import Status +from opentelemetry.trace import StatusCode + + +def record_exception_on_span(exc: BaseException, *, escaped: bool = False) -> None: + """Attach an exception to the active span and mark the span as errored. + + Exceptions that propagate out of a request are recorded automatically by the + FastAPI instrumentation, so this is only needed for exceptions that are + caught and turned into an error response without ever being re-raised — + those would otherwise leave the span looking like a success. + + Records ``exception.type``, ``exception.message`` and + ``exception.stacktrace`` as a span event, per the OpenTelemetry semantic + conventions. + """ + span = trace.get_current_span() + if not span.is_recording(): + return + + span.record_exception(exc, escaped=escaped) + span.set_status(Status(StatusCode.ERROR, str(exc))) diff --git a/wavefront/server/modules/common_module/pyproject.toml b/wavefront/server/modules/common_module/pyproject.toml index e2be91a8..02b507b9 100644 --- a/wavefront/server/modules/common_module/pyproject.toml +++ b/wavefront/server/modules/common_module/pyproject.toml @@ -13,7 +13,13 @@ dependencies = [ "dependency-injector>=4.42.0,<5.0.0", "apscheduler>=3.11.0,<4.0.0", "redis>=5.2.1,<6.0.0", - "prometheus-client>=0.22.1,<1.0.0" + "opentelemetry-api>=1.28.2,<2.0.0", + "opentelemetry-sdk>=1.28.2,<2.0.0", + "opentelemetry-exporter-otlp>=1.28.2,<2.0.0", + "opentelemetry-instrumentation-fastapi>=0.49b0", + "opentelemetry-instrumentation-sqlalchemy>=0.49b0", + "opentelemetry-instrumentation-redis>=0.49b0", + "opentelemetry-instrumentation-httpx>=0.49b0" ] [dependency-groups] diff --git a/wavefront/server/modules/common_module/tests/conftest.py b/wavefront/server/modules/common_module/tests/conftest.py index f9eb6955..a20c5c59 100644 --- a/wavefront/server/modules/common_module/tests/conftest.py +++ b/wavefront/server/modules/common_module/tests/conftest.py @@ -31,7 +31,7 @@ async def test_endpoint(request: Request): @app.get('/metrics') async def metrics_endpoint(): - """Mock metrics endpoint similar to /v1/_metrics.""" + """Mock metrics endpoint used to exercise middleware on a plain route.""" return {'metrics': 'mock_data'} @app.get('/error') diff --git a/wavefront/server/modules/db_repo_module/db_repo_module/database/connection.py b/wavefront/server/modules/db_repo_module/db_repo_module/database/connection.py index 89bc418f..f9b011f3 100644 --- a/wavefront/server/modules/db_repo_module/db_repo_module/database/connection.py +++ b/wavefront/server/modules/db_repo_module/db_repo_module/database/connection.py @@ -27,6 +27,11 @@ def __init__(self, db_config: DatabaseConfig) -> None: autocommit=False, bind=self._engine ) + @property + def engine(self): + """The underlying async engine, for callers such as instrumentation.""" + return self._engine + async def close(self): if self._engine is None: raise Exception('DatabaseClient is not initialized') diff --git a/wavefront/server/modules/user_management_module/user_management_module/authorization/require_auth.py b/wavefront/server/modules/user_management_module/user_management_module/authorization/require_auth.py index 74e81639..b5fb9369 100644 --- a/wavefront/server/modules/user_management_module/user_management_module/authorization/require_auth.py +++ b/wavefront/server/modules/user_management_module/user_management_module/authorization/require_auth.py @@ -41,7 +41,6 @@ '/floware/v1/user/send-reset-password-email', '/floware/v1/user/reset-password', '/floware/v1/data-sources/outlook/webhook/email_received', - '/v1/_metrics', '/floware/v1/plugin-auth/authenticate', '/floware/v1/oauth/google/callback', '/floware/v1/oauth/microsoft/callback', diff --git a/wavefront/server/uv.lock b/wavefront/server/uv.lock index c1558e74..858e9d2a 100644 --- a/wavefront/server/uv.lock +++ b/wavefront/server/uv.lock @@ -319,6 +319,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/8e/6b293f883fdbd29b9c8170db44bddff9e7de224d8cf1eb4287f69f1766e5/argcomplete-1.10.3-py2.py3-none-any.whl", hash = "sha256:d8ea63ebaec7f59e56e7b2a386b1d1c7f1a7ae87902c9ee17d377eaa557f06fa", size = 36576, upload-time = "2019-11-26T19:12:46.646Z" }, ] +[[package]] +name = "asgiref" +version = "3.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e6/26/3b59f2bdae5f640389becb1f673cded775287f5fc4f816309d9ca9a3f93d/asgiref-3.12.1.tar.gz", hash = "sha256:59dcb51c272ad209d59bed5708a64a333083e86017d7fcdd67498eeab7784340", size = 42378, upload-time = "2026-07-14T09:56:18.087Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/1b/54f4ad77cd8a584fa70746c47df988e002cf1ee1eba43364d46f87803647/asgiref-3.12.1-py3-none-any.whl", hash = "sha256:fe386d1c2bff7259ea95929266d12a8cf9a8b5a1c2598402967d8792e7a7c094", size = 25478, upload-time = "2026-07-14T09:56:16.926Z" }, +] + [[package]] name = "asn1crypto" version = "1.5.1" @@ -1009,7 +1018,13 @@ dependencies = [ { name = "dependency-injector" }, { name = "fastapi" }, { name = "loguru" }, - { name = "prometheus-client" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-httpx" }, + { name = "opentelemetry-instrumentation-redis" }, + { name = "opentelemetry-instrumentation-sqlalchemy" }, + { name = "opentelemetry-sdk" }, { name = "redis" }, ] @@ -1027,7 +1042,13 @@ requires-dist = [ { name = "dependency-injector", specifier = ">=4.42.0,<5.0.0" }, { name = "fastapi", specifier = ">=0.115.2,<1.0.0" }, { name = "loguru", specifier = ">=0.7.2,<1.0.0" }, - { name = "prometheus-client", specifier = ">=0.22.1,<1.0.0" }, + { name = "opentelemetry-api", specifier = ">=1.28.2,<2.0.0" }, + { name = "opentelemetry-exporter-otlp", specifier = ">=1.28.2,<2.0.0" }, + { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.49b0" }, + { name = "opentelemetry-instrumentation-httpx", specifier = ">=0.49b0" }, + { name = "opentelemetry-instrumentation-redis", specifier = ">=0.49b0" }, + { name = "opentelemetry-instrumentation-sqlalchemy", specifier = ">=0.49b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.28.2,<2.0.0" }, { name = "redis", specifier = ">=5.2.1,<6.0.0" }, ] @@ -3621,6 +3642,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d8/3e/f6f10f178b6316de67f0dfdbbb699a24fbe8917cf1743c1595fb9dcdd461/opentelemetry_instrumentation-0.61b0-py3-none-any.whl", hash = "sha256:92a93a280e69788e8f88391247cc530fd81f16f2b011979d4d6398f805cfbc63", size = 33448, upload-time = "2026-03-04T14:19:02.447Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-asgi" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/3e/143cf5c034e58037307e6a24f06e0dd64b2c49ae60a965fc580027581931/opentelemetry_instrumentation_asgi-0.61b0.tar.gz", hash = "sha256:9d08e127244361dc33976d39dd4ca8f128b5aa5a7ae425208400a80a095019b5", size = 26691, upload-time = "2026-03-04T14:20:21.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/78/154470cf9d741a7487fbb5067357b87386475bbb77948a6707cae982e158/opentelemetry_instrumentation_asgi-0.61b0-py3-none-any.whl", hash = "sha256:e4b3ce6b66074e525e717efff20745434e5efd5d9df6557710856fba356da7a4", size = 16980, upload-time = "2026-03-04T14:19:10.894Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-fastapi" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/35/aa727bb6e6ef930dcdc96a617b83748fece57b43c47d83ba8d83fbeca657/opentelemetry_instrumentation_fastapi-0.61b0.tar.gz", hash = "sha256:3a24f35b07c557ae1bbc483bf8412221f25d79a405f8b047de8b670722e2fa9f", size = 24800, upload-time = "2026-03-04T14:20:32.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/05/acfeb2cccd434242a0a7d0ea29afaf077e04b42b35b485d89aee4e0d9340/opentelemetry_instrumentation_fastapi-0.61b0-py3-none-any.whl", hash = "sha256:a1a844d846540d687d377516b2ff698b51d87c781b59f47c214359c4a241047c", size = 13485, upload-time = "2026-03-04T14:19:30.351Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-httpx" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/2a/e2becd55e33c29d1d9ef76e2579040ed1951cb33bacba259f6aff2fdd2a6/opentelemetry_instrumentation_httpx-0.61b0.tar.gz", hash = "sha256:6569ec097946c5551c2a4252f74c98666addd1bf047c1dde6b4ef426719ff8dd", size = 24104, upload-time = "2026-03-04T14:20:34.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/88/dde310dce56e2d85cf1a09507f5888544955309edc4b8d22971d6d3d1417/opentelemetry_instrumentation_httpx-0.61b0-py3-none-any.whl", hash = "sha256:dee05c93a6593a5dc3ae5d9d5c01df8b4e2c5d02e49275e5558534ee46343d5e", size = 17198, upload-time = "2026-03-04T14:19:33.585Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-redis" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/21/26205f89358a5f2be3ee5512d3d3bce16b622977f64aeaa9d3fa8887dd39/opentelemetry_instrumentation_redis-0.61b0.tar.gz", hash = "sha256:ae0fbb56be9a641e621d55b02a7d62977a2c77c5ee760addd79b9b266e46e523", size = 14781, upload-time = "2026-03-04T14:20:45.694Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/e1/8f4c8e4194291dbe828aeabe779050a8497b379ad90040a5a0a7074b1d08/opentelemetry_instrumentation_redis-0.61b0-py3-none-any.whl", hash = "sha256:8d4e850bbb5f8eeafa44c0eac3a007990c7125de187bc9c3659e29ff7e091172", size = 15506, upload-time = "2026-03-04T14:19:48.588Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-sqlalchemy" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/4f/3a325b180944610697a0a926d49d782b41a86120050d44fefb2715b630ac/opentelemetry_instrumentation_sqlalchemy-0.61b0.tar.gz", hash = "sha256:13a3a159a2043a52f0180b3757fbaa26741b0e08abb50deddce4394c118956e6", size = 15343, upload-time = "2026-03-04T14:20:47.648Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/97/b906a930c6a1a20c53ecc8b58cabc2cdd0ce560a2b5d44259084ffe4333e/opentelemetry_instrumentation_sqlalchemy-0.61b0-py3-none-any.whl", hash = "sha256:f115e0be54116ba4c327b8d7b68db4045ee18d44439d888ab8130a549c50d1c1", size = 14547, upload-time = "2026-03-04T14:19:53.088Z" }, +] + [[package]] name = "opentelemetry-proto" version = "1.40.0" @@ -3660,6 +3760,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/37/cc6a55e448deaa9b27377d087da8615a3416d8ad523d5960b78dbeadd02a/opentelemetry_semantic_conventions-0.61b0-py3-none-any.whl", hash = "sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2", size = 231621, upload-time = "2026-03-04T14:17:19.33Z" }, ] +[[package]] +name = "opentelemetry-util-http" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/3c/f0196223efc5c4ca19f8fad3d5462b171ac6333013335ce540c01af419e9/opentelemetry_util_http-0.61b0.tar.gz", hash = "sha256:1039cb891334ad2731affdf034d8fb8b48c239af9b6dd295e5fabd07f1c95572", size = 11361, upload-time = "2026-03-04T14:20:57.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/e5/c08aaaf2f64288d2b6ef65741d2de5454e64af3e050f34285fb1907492fe/opentelemetry_util_http-0.61b0-py3-none-any.whl", hash = "sha256:8e715e848233e9527ea47e275659ea60a57a75edf5206a3b937e236a6da5fc33", size = 9281, upload-time = "2026-03-04T14:20:08.364Z" }, +] + [[package]] name = "orjson" version = "3.11.4" @@ -4084,15 +4193,6 @@ requires-dist = [ { name = "db-repo-module", editable = "modules/db_repo_module" }, ] -[[package]] -name = "prometheus-client" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, -] - [[package]] name = "prompt-toolkit" version = "3.0.52"