From 59e39a7ccbb063ad902e6e7c3500eb4fe0261444 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 29 Aug 2026 15:02:11 +0200 Subject: [PATCH 01/19] feat(audit): org-wide audit log with actor attribution and durable queue delivery Adds an append-only audit trail distinguishing users, API keys, agents, and system automation, following the auditlog.dev spec: - audit_log_entries table (migration 0050): actor snapshot + credential refs, outcome (allowed/denied) + denial_reason, before/after change diffs with queryable changed_fields, affected_user, request forensics (request id, origin IP/country), and occurred_at/recorded_at. - Durable delivery through a Cloudflare Queue (audit-events): producers enqueue, the api worker consumes and inserts idempotently; direct DB write as fallback when the binding is absent or the send fails. - CurrentAuditActor context from all three auth layers distinguishes session vs API-key requests; denied attempts (scope/org/surface rejections) are recorded from inside the auth layers. - Recording wired into every v2 mutation handler (with diffs and secret redaction), the issue-workflow choke point (agent vs user attribution with on-behalf-of), and register_agent. - GET /v2/audit_log: cursor-paginated, filterable by actor, outcome, action, resource, changed field, request id, and time window; new audit_log:read scope and alog_ public IDs. - Settings > Audit Log tab with actor/outcome filters, denied badges, change summaries, and load-more pagination. - Hourly retention sweep (AUDIT_LOG_RETENTION_DAYS, default 400) in the api worker's existing retention cron. --- apps/alerting/src/worker.ts | 5 +- apps/api/alchemy.run.ts | 20 + apps/api/src/alerting.ts | 1 + apps/api/src/audit-events-runtime.ts | 70 + apps/api/src/mcp/tools/register-agent.ts | 13 + .../api/src/mcp/tools/runtime-requirements.ts | 2 + apps/api/src/queue-dispatch.ts | 5 +- .../v2/alchemy-provider.integration.test.ts | 2 + .../src/routes/v2/alert-destinations.http.ts | 83 +- apps/api/src/routes/v2/alert-rules.http.ts | 67 +- apps/api/src/routes/v2/alerts.http.test.ts | 2 + apps/api/src/routes/v2/anomalies.http.ts | 16 +- apps/api/src/routes/v2/api-keys.http.test.ts | 2 + apps/api/src/routes/v2/api-keys.http.ts | 18 + .../src/routes/v2/attribute-mappings.http.ts | 31 +- apps/api/src/routes/v2/audit-changes.ts | 60 + apps/api/src/routes/v2/audit-log.http.ts | 141 + .../routes/v2/config-resources.http.test.ts | 2 + .../api/src/routes/v2/dashboards.http.test.ts | 2 + apps/api/src/routes/v2/dashboards.http.ts | 76 +- apps/api/src/routes/v2/ingest-keys.http.ts | 9 + .../src/routes/v2/integrations.http.test.ts | 2 + .../src/routes/v2/mobile-devices.http.test.ts | 2 + .../routes/v2/phase1-resources.http.test.ts | 2 + apps/api/src/routes/v2/scrape-targets.http.ts | 67 +- .../src/routes/v2/setup-audit.http.test.ts | 2 + apps/api/src/routes/v2/telemetry.http.test.ts | 2 + apps/api/src/routes/v2/v2-test-support.ts | 8 + .../routes/v2/widget-credentials.http.test.ts | 2 + .../src/routes/v2/widget-summary.http.test.ts | 2 + apps/api/src/runtime/graph-boundaries.test.ts | 2 + apps/api/src/runtime/http-graph.ts | 5 + apps/api/src/runtime/mcp-service-graph.ts | 4 +- apps/api/src/runtime/service-graph.ts | 3 + .../services/audit/AuditLogService.test.ts | 202 + .../api/src/services/audit/AuditLogService.ts | 291 + apps/api/src/services/audit/audit-event.ts | 62 + .../src/services/audit/audit-log-retention.ts | 78 + .../services/auth/ApiAuthorizationLayer.ts | 36 +- .../services/auth/ApiAuthorizationV2Layer.ts | 61 +- .../auth/SessionAuthorizationLayer.ts | 8 +- apps/api/src/services/auth/audit-actor.ts | 23 + .../ErrorIssueReadModelsService.test.ts | 7 +- .../errors/ErrorIssueWorkflowService.test.ts | 11 +- .../errors/ErrorIssueWorkflowService.ts | 95 +- .../src/services/errors/ErrorsService.test.ts | 3 + .../IssueFixVerificationService.test.ts | 2 + apps/api/src/vcs-sync-runtime.ts | 5 +- apps/api/src/worker.ts | 23 +- apps/api/wrangler.jsonc | 8 + .../components/settings/audit-log-section.tsx | 337 + .../src/components/settings/settings-nav.tsx | 4 + .../src/lib/services/atoms/audit-log-atoms.ts | 46 + apps/web/src/routes/settings.tsx | 2 + .../db/drizzle/0050_audit_log_entries.sql | 31 + packages/db/drizzle/meta/0050_snapshot.json | 8999 +++++++++++++++++ packages/db/drizzle/meta/_journal.json | 9 +- packages/db/src/schema/audit-log.ts | 61 + packages/db/src/schema/index.ts | 1 + packages/domain/src/http/audit-log.ts | 54 + packages/domain/src/http/index.ts | 1 + packages/domain/src/http/v2/api.ts | 2 + packages/domain/src/http/v2/audit-log.ts | 246 + packages/domain/src/http/v2/index.ts | 1 + packages/domain/src/http/v2/openapi.test.ts | 1 + packages/domain/src/http/v2/public-id.ts | 1 + packages/primitives/src/index.ts | 3 + 67 files changed, 11387 insertions(+), 57 deletions(-) create mode 100644 apps/api/src/audit-events-runtime.ts create mode 100644 apps/api/src/routes/v2/audit-changes.ts create mode 100644 apps/api/src/routes/v2/audit-log.http.ts create mode 100644 apps/api/src/services/audit/AuditLogService.test.ts create mode 100644 apps/api/src/services/audit/AuditLogService.ts create mode 100644 apps/api/src/services/audit/audit-event.ts create mode 100644 apps/api/src/services/audit/audit-log-retention.ts create mode 100644 apps/api/src/services/auth/audit-actor.ts create mode 100644 apps/web/src/components/settings/audit-log-section.tsx create mode 100644 apps/web/src/lib/services/atoms/audit-log-atoms.ts create mode 100644 packages/db/drizzle/0050_audit_log_entries.sql create mode 100644 packages/db/drizzle/meta/0050_snapshot.json create mode 100644 packages/db/src/schema/audit-log.ts create mode 100644 packages/domain/src/http/audit-log.ts create mode 100644 packages/domain/src/http/v2/audit-log.ts diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index 0b91bfc76..d5608448e 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -6,6 +6,7 @@ import { AlertRulesService, AlertsService, AnomalyDetectionService, + AuditLogService, BucketCacheService, CacheBackendLive, CloudflareAnalyticsService, @@ -144,7 +145,9 @@ export const buildLayer = (env: AlertingWorkerEnv) => { const ErrorActorsServiceLive = ErrorActorsService.layer.pipe(Layer.provide(BaseLive)) const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(Layer.mergeAll(BaseLive, ErrorActorsServiceLive)), + Layer.provide( + Layer.mergeAll(BaseLive, ErrorActorsServiceLive, AuditLogService.layer.pipe(Layer.provide(BaseLive))), + ), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer.pipe(Layer.provide(BaseLive)) const ErrorIssueReadModelsServiceLive = ErrorIssueReadModelsService.layer.pipe( diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index d0f692a33..162b5862a 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -178,6 +178,8 @@ const apiConfiguredEnv = (stage: MapleStage) => // Agent LLM path. `MAPLE_LLM_PROVIDER` flips between OpenRouter (default) and // Workers AI; both stay wired, so a switch is this one var plus a redeploy. // See `@/platform/Llm` for the provider-scoped model overrides. + // Audit log retention horizon in days; the sweep defaults to 400 when unset. + optionalPlain("AUDIT_LOG_RETENTION_DAYS"), optionalPlain("MAPLE_LLM_PROVIDER"), optionalPlain("MAPLE_TRIAGE_MODEL_OPENROUTER"), optionalPlain("MAPLE_TRIAGE_MODEL_WORKERS_AI"), @@ -318,6 +320,10 @@ export const createMapleApi = ({ stage, domains, replayBlobs }: CreateMapleApiOp const planetScaleWebhookQueue = yield* Cloudflare.Queues.Queue("planetscale-webhooks", { name: planetScaleWebhookQueueName, }) + const auditEventsQueueName = resolveWorkerName("audit-events", stage) + const auditEventsQueue = yield* Cloudflare.Queues.Queue("audit-events", { + name: auditEventsQueueName, + }) const worker = (yield* Cloudflare.Worker("api", { name: resolveWorkerName("api", stage), @@ -369,6 +375,8 @@ export const createMapleApi = ({ stage, domains, replayBlobs }: CreateMapleApiOp VCS_SYNC_QUEUE_NAME: vcsSyncQueueName, PLANETSCALE_WEBHOOK_QUEUE: planetScaleWebhookQueue, PLANETSCALE_WEBHOOK_QUEUE_NAME: planetScaleWebhookQueueName, + AUDIT_EVENTS_QUEUE: auditEventsQueue, + AUDIT_EVENTS_QUEUE_NAME: auditEventsQueueName, CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: schemaApplyWorkflow, INVESTIGATION_FANOUT_WORKFLOW: investigationFanoutWorkflow, API_V2_RATE_LIMITER: Cloudflare.RateLimit("API_V2_RATE_LIMITER", { @@ -428,6 +436,18 @@ export const createMapleApi = ({ stage, domains, replayBlobs }: CreateMapleApiOp maxWaitTimeMs: 5000, }, }) + // Audit entries tolerate a few seconds of delivery latency; batch wider and + // wait longer so one insert round-trip covers many entries. + yield* Cloudflare.Queues.Consumer("audit-events-consumer", { + queueId: auditEventsQueue.queueId, + scriptName: worker.workerName, + settings: { + batchSize: 25, + maxConcurrency: 2, + maxRetries: 5, + maxWaitTimeMs: 5000, + }, + }) // `db` is undefined on ref stages — alerting resolves the same ref itself. return { worker, db: mapleDb } diff --git a/apps/api/src/alerting.ts b/apps/api/src/alerting.ts index f822ab0f0..230952228 100644 --- a/apps/api/src/alerting.ts +++ b/apps/api/src/alerting.ts @@ -4,6 +4,7 @@ export { AlertDestinationsService } from "./services/alerts/AlertDestinationsSer export { AlertReadModelsService } from "./services/alerts/AlertReadModelsService" export { AlertRulesService } from "./services/alerts/AlertRulesService" export { AnomalyDetectionService } from "./services/alerts/AnomalyDetectionService" +export { AuditLogService } from "./services/audit/AuditLogService" export { BucketCacheService } from "@maple/query-engine/caching" export { CacheBackendLive } from "@/platform/CacheBackendLive" export { CloudflareAnalyticsService } from "./services/integrations/CloudflareAnalyticsService" diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts new file mode 100644 index 000000000..b7b18d063 --- /dev/null +++ b/apps/api/src/audit-events-runtime.ts @@ -0,0 +1,70 @@ +import type { MessageBatch } from "@cloudflare/workers-types" +import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" +import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" +import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" +import { auditLogEntries } from "@maple/db" +import { Clock, Effect, Layer } from "effect" +import { layerPg } from "@/platform/DatabasePgLive" +import { Database } from "@/platform/DatabaseLive" +import { auditEventToInsert, decodeAuditLogEvent } from "./services/audit/audit-event" + +const telemetry = MapleCloudflareSDK.make({ + serviceName: "maple-api", + serviceNamespace: "core", + repositoryUrl: "https://github.com/MapleTechLabs/maple", + anticipatedErrorIdentifiers: [...ANTICIPATED_ERROR_IDENTIFIERS], +}) + +export const buildAuditEventsLayer = (_env: Record) => { + const DatabaseLive = layerPg.pipe(Layer.provide(WorkerEnvironment.layer)) + return DatabaseLive.pipe( + Layer.provideMerge(telemetry.layer), + Layer.provideMerge(WorkerEnvironment.layer), + Layer.provideMerge(WorkerConfigProviderLayer), + ) +} + +export const flushAuditEventsTelemetry = (env: Record) => telemetry.flush(env) + +/** + * Audit events queue consumer: lowers each event to its `audit_log_entries` + * row. The `(org_id, id)` primary key plus `onConflictDoNothing` makes queue + * redelivery idempotent; insert failures retry through the queue's policy. + */ +export const processAuditEventsBatch = (batch: MessageBatch) => + Effect.gen(function* () { + const database = yield* Database + yield* Effect.forEach( + batch.messages, + (message) => + decodeAuditLogEvent(message.body).pipe( + Effect.matchEffect({ + onFailure: (error) => + Effect.logWarning("Discarding malformed audit event queue message").pipe( + Effect.annotateLogs({ attempt: message.attempts, error: String(error) }), + Effect.flatMap(() => Effect.sync(() => message.ack())), + ), + onSuccess: (event) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + yield* database.execute((db) => + db + .insert(auditLogEntries) + .values(auditEventToInsert(event, now)) + .onConflictDoNothing(), + ) + yield* Effect.sync(() => message.ack()) + }).pipe( + Effect.withSpan("auditEvents.processMessage"), + Effect.catchCause((cause) => + Effect.logWarning("Audit event insert failed; retrying").pipe( + Effect.annotateLogs({ attempt: message.attempts, error: String(cause) }), + Effect.flatMap(() => Effect.sync(() => message.retry())), + ), + ), + ), + }), + ), + { concurrency: 5, discard: true }, + ) + }).pipe(Effect.withSpan("auditEvents.processBatch")) diff --git a/apps/api/src/mcp/tools/register-agent.ts b/apps/api/src/mcp/tools/register-agent.ts index be1f258cb..1e887c067 100644 --- a/apps/api/src/mcp/tools/register-agent.ts +++ b/apps/api/src/mcp/tools/register-agent.ts @@ -5,9 +5,11 @@ import { validationError, type McpToolRegistrar, } from "./types" +import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" import { Effect, Option, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { CurrentMcpTenant } from "@/mcp/lib/query-warehouse" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ErrorActorsService } from "@/services/errors/ErrorActorsService" const decodeStringArray = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Array(Schema.String))) @@ -58,6 +60,17 @@ export function registerRegisterAgentTool(server: McpToolRegistrar) { ), ) + const audit = yield* AuditLogService + yield* audit.record({ + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: "mcp", + action: "agent.registered", + resourceType: "agent", + resourceId: encodePublicId(PublicIdPrefixes.actor, actor.id), + metadata: { name: actor.agentName ?? name }, + }) + const lines = [ `## Agent registered`, `- Actor ID: ${actor.id}`, diff --git a/apps/api/src/mcp/tools/runtime-requirements.ts b/apps/api/src/mcp/tools/runtime-requirements.ts index baf795ab8..8b4e1f3e3 100644 --- a/apps/api/src/mcp/tools/runtime-requirements.ts +++ b/apps/api/src/mcp/tools/runtime-requirements.ts @@ -1,3 +1,4 @@ +import type { AuditLogService } from "@/services/audit/AuditLogService" import type { AlertsService } from "@/services/alerts/AlertsService" import type { AlertReadModelsService } from "@/services/alerts/AlertReadModelsService" import type { AlertRulesService } from "@/services/alerts/AlertRulesService" @@ -22,6 +23,7 @@ import type { CurrentMcpTenant } from "../lib/query-warehouse" */ export type McpToolRuntimeRequirements = | AlertsService + | AuditLogService | AlertReadModelsService | AlertRulesService | DashboardPersistenceService diff --git a/apps/api/src/queue-dispatch.ts b/apps/api/src/queue-dispatch.ts index 77f3e320e..44502e140 100644 --- a/apps/api/src/queue-dispatch.ts +++ b/apps/api/src/queue-dispatch.ts @@ -1,4 +1,4 @@ -export type WorkerQueueKind = "planetscale-webhook" | "vcs-sync" | "unknown" +export type WorkerQueueKind = "planetscale-webhook" | "vcs-sync" | "audit-events" | "unknown" export const classifyWorkerQueue = (queueName: string, env: Record): WorkerQueueKind => { if ( @@ -10,5 +10,8 @@ export const classifyWorkerQueue = (queueName: string, env: Record { Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/alert-destinations.http.ts b/apps/api/src/routes/v2/alert-destinations.http.ts index ea2660a9d..73238c668 100644 --- a/apps/api/src/routes/v2/alert-destinations.http.ts +++ b/apps/api/src/routes/v2/alert-destinations.http.ts @@ -1,5 +1,5 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" -import type { AlertDestinationDocument, AlertDestinationUpdateRequest } from "@maple/domain/http" +import type { AlertDestinationDocument, AlertDestinationUpdateRequest, AuditChanges } from "@maple/domain/http" import { CurrentTenant, DiscordAlertDestinationConfig, @@ -18,8 +18,9 @@ import type { V2AlertDestinationUpdateParams, V2TelegramChatList, } from "@maple/domain/http/v2" -import { MapleApiV2, paginateArray } from "@maple/domain/http/v2" +import { encodePublicId, MapleApiV2, paginateArray, PublicIdPrefixes } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { AlertDestinationsService } from "@/services/alerts/AlertDestinationsService" const toV2Destination = (doc: AlertDestinationDocument): V2AlertDestination => ({ @@ -191,6 +192,61 @@ const toUpdateRequest = (params: V2AlertDestinationUpdateParams): AlertDestinati } } +/** Credential-bearing config keys; their values must never reach the audit row. */ +const destinationSecretKeys = new Set(["integrationKey", "signingSecret", "url", "webhookUrl", "botToken"]) + +/** Fields of an update that are readable back off the destination document. */ +const destinationObservableValue = (doc: AlertDestinationDocument, key: string): unknown => { + switch (key) { + case "name": + return doc.name + case "enabled": + return doc.enabled + case "memberUserIds": + return doc.memberUserIds + default: + return undefined + } +} + +/** + * Diff an update against the pre/post documents. Secrets are recorded as + * ``; config knobs the wire doc doesn't echo (channel ids, chat ids) + * are recorded as touched with `` placeholders. + */ +const buildDestinationChanges = ( + request: AlertDestinationUpdateRequest, + before: AlertDestinationDocument | undefined, + after: AlertDestinationDocument, +): AuditChanges | undefined => { + const fields: string[] = [] + const beforeOut: Record = {} + const afterOut: Record = {} + for (const key of Object.keys(request)) { + if (key === "type") continue + const wireName = key.replace(/[A-Z]/g, (char) => `_${char.toLowerCase()}`) + if (destinationSecretKeys.has(key)) { + fields.push(wireName) + beforeOut[wireName] = "" + afterOut[wireName] = "" + continue + } + const prev = before === undefined ? undefined : destinationObservableValue(before, key) + const next = destinationObservableValue(after, key) + if (prev === undefined && next === undefined) { + fields.push(wireName) + beforeOut[wireName] = "" + afterOut[wireName] = "" + continue + } + if (JSON.stringify(prev) === JSON.stringify(next)) continue + fields.push(wireName) + beforeOut[wireName] = prev + afterOut[wireName] = next + } + return fields.length === 0 ? undefined : { fields, before: beforeOut, after: afterOut } +} + export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "alertDestinations", (handlers) => Effect.gen(function* () { const destinations = yield* AlertDestinationsService @@ -241,20 +297,37 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale toCreateRequest(payload), ) + yield* recordHttpAudit("alert_destination.created", { + resourceType: "alert_destination", + resourceId: encodePublicId(PublicIdPrefixes.alertDestination, created.id), + metadata: { name: created.name, type: created.type }, + }) + return toV2DestinationMutation(created) }), ) .handle("update", ({ params, payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + const request = toUpdateRequest(payload) + const existing = yield* destinations.listDestinations(tenant.orgId) + const current = existing.destinations.find((doc) => doc.id === params.id) const updated = yield* destinations.updateDestination( tenant.orgId, tenant.userId, tenant.roles, params.id, - toUpdateRequest(payload), + request, ) + const changes = buildDestinationChanges(request, current, updated) + yield* recordHttpAudit("alert_destination.updated", { + resourceType: "alert_destination", + resourceId: encodePublicId(PublicIdPrefixes.alertDestination, updated.id), + ...(changes !== undefined ? { changes } : undefined), + metadata: { name: updated.name, type: updated.type }, + }) + return toV2DestinationMutation(updated) }), ) @@ -266,6 +339,10 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale tenant.roles, params.id, ) + yield* recordHttpAudit("alert_destination.deleted", { + resourceType: "alert_destination", + resourceId: encodePublicId(PublicIdPrefixes.alertDestination, deleted.id), + }) return { id: deleted.id, diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 73d7e3d84..54c61591c 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -16,9 +16,19 @@ import type { V2AlertRulePreviewResult, V2AlertRuleUpdateParams, } from "@maple/domain/http/v2" -import { MapleApiV2, paginateArray, scopeAllows, timestamp, V2ParameterInvalid } from "@maple/domain/http/v2" +import { + encodePublicId, + MapleApiV2, + paginateArray, + PublicIdPrefixes, + scopeAllows, + timestamp, + V2ParameterInvalid, +} from "@maple/domain/http/v2" import { AlertForbiddenError } from "@maple/domain/http" import { Effect, Encoding, Result, Schema } from "effect" +import { compactAuditChanges, diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { AlertsService } from "@/services/alerts/AlertsService" import { AlertReadModelsService } from "@/services/alerts/AlertReadModelsService" import { AlertRulesService } from "@/services/alerts/AlertRulesService" @@ -94,6 +104,37 @@ const toV2Rule = (doc: AlertRuleDocument): V2AlertRule => ({ updated_by: doc.updatedBy, }) +/** Update-payload fields diffable through the wire shape (drafts get summarized). */ +const ruleAuditKeys: ReadonlyArray = [ + "name", + "notes", + "notification_template", + "enabled", + "severity", + "service_names", + "exclude_service_names", + "environments", + "tags", + "group_by", + "signal_type", + "comparator", + "threshold", + "threshold_upper", + "window_minutes", + "minimum_sample_count", + "consecutive_breaches_required", + "consecutive_healthy_required", + "renotify_interval_minutes", + "apdex_threshold_ms", + "query_builder_draft", + "raw_query_sql", + "raw_query_reducer", + "destination_ids", +] + +/** Query drafts and raw SQL are config blobs — audit that they changed, not their bodies. */ +const summarizeRuleBlob = (value: unknown) => (value === null ? null : "") + const toV2RuleMutationResponse = (doc: AlertRuleDocument): V2AlertRuleMutationResponse => ({ ...toV2Rule(doc), ...(doc.txid !== undefined ? { txid: doc.txid } : undefined), @@ -345,6 +386,12 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules request, ) + yield* recordHttpAudit("alert_rule.created", { + resourceType: "alert_rule", + resourceId: encodePublicId(PublicIdPrefixes.alertRule, created.id), + metadata: { name: created.name }, + }) + return toV2RuleMutationResponse(created) }), ) @@ -361,6 +408,20 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules request, ) + const changes = compactAuditChanges( + diffAuditChanges( + pickPresentFields(ruleAuditKeys, payload, toV2Rule(current)), + pickPresentFields(ruleAuditKeys, payload, toV2Rule(updated)), + ), + { query_builder_draft: summarizeRuleBlob, raw_query_sql: summarizeRuleBlob }, + ) + yield* recordHttpAudit("alert_rule.updated", { + resourceType: "alert_rule", + resourceId: encodePublicId(PublicIdPrefixes.alertRule, updated.id), + ...(changes !== undefined ? { changes } : undefined), + metadata: { name: updated.name }, + }) + return toV2RuleMutationResponse(updated) }), ) @@ -368,6 +429,10 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* rules.deleteRule(tenant.orgId, tenant.roles, params.id) + yield* recordHttpAudit("alert_rule.deleted", { + resourceType: "alert_rule", + resourceId: encodePublicId(PublicIdPrefixes.alertRule, deleted.id), + }) return { id: deleted.id, diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 418a76f00..da3466686 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -18,6 +18,7 @@ import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@/platfor import type { WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQueryService" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -164,6 +165,7 @@ const makeHarness = ( Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/anomalies.http.ts b/apps/api/src/routes/v2/anomalies.http.ts index ed51bf9d1..1cfc5dad2 100644 --- a/apps/api/src/routes/v2/anomalies.http.ts +++ b/apps/api/src/routes/v2/anomalies.http.ts @@ -12,9 +12,10 @@ import { AnomalyForbiddenError, CurrentTenant, } from "@maple/domain/http" -import { MapleApiV2, paginateOffsetQuery, timestamp } from "@maple/domain/http/v2" +import { encodePublicId, MapleApiV2, paginateOffsetQuery, PublicIdPrefixes, timestamp } from "@maple/domain/http/v2" import type { V2AnomalyIncident, V2AnomalyIncidentTimeseries, V2AnomalySettings } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { requireAdmin } from "@/services/auth/auth" import { AnomalyDetectionService } from "@/services/alerts/AnomalyDetectionService" import { ErrorsService } from "@/services/errors/ErrorsService" @@ -186,6 +187,14 @@ export const HttpV2AnomaliesLive = HttpApiBuilder.group(MapleApiV2, "anomalies", Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const incident = yield* anomalies.resolveIncidentManually(tenant.orgId, params.id) + yield* recordHttpAudit("anomaly_incident.resolved", { + resourceType: "anomaly_incident", + resourceId: encodePublicId(PublicIdPrefixes.anomalyIncident, incident.id), + metadata: { + signal_type: incident.signalType, + service_name: incident.serviceName, + }, + }) return toV2Incident(incident) }), @@ -244,6 +253,11 @@ export const HttpV2AnomaliesLive = HttpApiBuilder.group(MapleApiV2, "anomalies", }), ) + yield* recordHttpAudit("anomaly_settings.updated", { + resourceType: "anomaly_settings", + metadata: { enabled: settings.enabled, sensitivity: settings.sensitivity }, + }) + return toV2Settings(settings) }), ) diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index b73d59b7c..b2da944b7 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -12,6 +12,7 @@ import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiV2RateLimiter, type ApiV2RateLimiterApi } from "@/services/auth/ApiV2RateLimiter" import { V2TransportErrorBoundaryLive } from "./error-envelope" import { @@ -69,6 +70,7 @@ const makeHarness = ( Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(Layer.succeed(ApiV2RateLimiter, { check: checkRateLimit })), Layer.provideMerge(servicesLive), Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS)), diff --git a/apps/api/src/routes/v2/api-keys.http.ts b/apps/api/src/routes/v2/api-keys.http.ts index 581de9352..152742d7d 100644 --- a/apps/api/src/routes/v2/api-keys.http.ts +++ b/apps/api/src/routes/v2/api-keys.http.ts @@ -2,14 +2,17 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import type { ApiKeyCreatedResponse, ApiKeyResponse } from "@maple/domain/http" import { CurrentTenant } from "@maple/domain/http" import { + encodePublicId, MapleApiV2, isoTimestamp, isoTimestampOrNull, paginateArray, + PublicIdPrefixes, V2InsufficientPermissions, } from "@maple/domain/http/v2" import type { V2ApiKey, V2ApiKeyMutationResponse, V2ApiKeyWithSecret } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { requireAdmin } from "@/services/auth/auth" @@ -106,6 +109,11 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha ? { metadataJson: { source: "maple_mcp", roles: [...tenant.roles] } } : undefined), }) + yield* recordHttpAudit("api_key.created", { + resourceType: "api_key", + resourceId: encodePublicId(PublicIdPrefixes.apiKey, created.id), + metadata: { name: created.name, kind: created.kind, scopes: created.scopes }, + }) return toV2ApiKeyWithSecret(created) }), ) @@ -117,6 +125,11 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha const rolled = yield* apiKeysService.roll(tenant.orgId, tenant.userId, params.id, { createdByEmail, }) + yield* recordHttpAudit("api_key.rolled", { + resourceType: "api_key", + resourceId: encodePublicId(PublicIdPrefixes.apiKey, rolled.id), + metadata: { name: rolled.name, scopes: rolled.scopes }, + }) return toV2ApiKeyWithSecret(rolled) }), ) @@ -132,6 +145,11 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha yield* requireAdmin(tenant.roles, adminOnly("revoke")) } const revoked = yield* apiKeysService.revoke(tenant.orgId, params.id) + yield* recordHttpAudit("api_key.revoked", { + resourceType: "api_key", + resourceId: encodePublicId(PublicIdPrefixes.apiKey, revoked.id), + metadata: { name: revoked.name }, + }) return toV2ApiKeyMutationResponse(revoked) }), ) diff --git a/apps/api/src/routes/v2/attribute-mappings.http.ts b/apps/api/src/routes/v2/attribute-mappings.http.ts index 3ff480b7f..4557de176 100644 --- a/apps/api/src/routes/v2/attribute-mappings.http.ts +++ b/apps/api/src/routes/v2/attribute-mappings.http.ts @@ -6,9 +6,11 @@ import { IngestAttributeMappingNotFoundError, UpdateIngestAttributeMappingRequest, } from "@maple/domain/http" -import { MapleApiV2, paginateArray } from "@maple/domain/http/v2" +import { encodePublicId, MapleApiV2, paginateArray, PublicIdPrefixes } from "@maple/domain/http/v2" import type { V2AttributeMapping } from "@maple/domain/http/v2" import { Array as Arr, Effect, Option } from "effect" +import { diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { IngestAttributeMappingService } from "@/services/org/IngestAttributeMappingService" const toV2AttributeMapping = (mapping: IngestAttributeMapping): V2AttributeMapping => ({ @@ -24,6 +26,11 @@ const toV2AttributeMapping = (mapping: IngestAttributeMapping): V2AttributeMappi updated_at: mapping.updatedAt, }) +/** Update-payload fields that are diffable through the wire shape. */ +const mappingAuditKeys: ReadonlyArray< + "name" | "source_context" | "source_key" | "target_key" | "operation" | "enabled" +> = ["name", "source_context", "source_key", "target_key", "operation", "enabled"] + export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "attributeMappings", (handlers) => Effect.gen(function* () { const service = yield* IngestAttributeMappingService @@ -80,12 +87,19 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att }), ) + yield* recordHttpAudit("attribute_mapping.created", { + resourceType: "attribute_mapping", + resourceId: encodePublicId(PublicIdPrefixes.attributeMapping, created.id), + metadata: { name: created.name }, + }) + return toV2AttributeMapping(created) }), ) .handle("update", ({ params, payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + const current = yield* findMapping(tenant.orgId, params.id) const updated = yield* service.update( tenant.orgId, params.id, @@ -109,6 +123,17 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att }), ) + const changes = diffAuditChanges( + pickPresentFields(mappingAuditKeys, payload, toV2AttributeMapping(current)), + pickPresentFields(mappingAuditKeys, payload, toV2AttributeMapping(updated)), + ) + yield* recordHttpAudit("attribute_mapping.updated", { + resourceType: "attribute_mapping", + resourceId: encodePublicId(PublicIdPrefixes.attributeMapping, updated.id), + ...(changes !== undefined ? { changes } : undefined), + metadata: { name: updated.name }, + }) + return toV2AttributeMapping(updated) }), ) @@ -116,6 +141,10 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* service.delete(tenant.orgId, params.id) + yield* recordHttpAudit("attribute_mapping.deleted", { + resourceType: "attribute_mapping", + resourceId: encodePublicId(PublicIdPrefixes.attributeMapping, deleted.id), + }) return { id: deleted.id, object: "attribute_mapping" as const, deleted: true as const } }), diff --git a/apps/api/src/routes/v2/audit-changes.ts b/apps/api/src/routes/v2/audit-changes.ts new file mode 100644 index 000000000..afa070f61 --- /dev/null +++ b/apps/api/src/routes/v2/audit-changes.ts @@ -0,0 +1,60 @@ +import type { AuditChanges } from "@maple/domain/http" + +/** + * Diff two snapshots restricted to the keys of `after` (the fields the request + * actually touched — omitted fields are unchanged by contract). Returns + * undefined when nothing changed so the audit entry can omit `changes`. + */ +export const diffAuditChanges = ( + before: Record, + after: Record, +): AuditChanges | undefined => { + const fields: string[] = [] + const beforeOut: Record = {} + const afterOut: Record = {} + for (const key of Object.keys(after)) { + const prev = before[key] + const next = after[key] + if (JSON.stringify(prev) === JSON.stringify(next)) continue + fields.push(key) + beforeOut[key] = prev + afterOut[key] = next + } + return fields.length === 0 ? undefined : { fields, before: beforeOut, after: afterOut } +} + +/** + * Snapshot only the fields the update payload actually carries, reading their + * values from a wire-shaped view of the resource (pre- or post-update). + */ +export const pickPresentFields = ( + keys: ReadonlyArray, + payload: { readonly [P in K]?: unknown }, + source: { readonly [P in K]: unknown }, +): Record => { + const out: Record = {} + for (const key of keys) { + if (payload[key] !== undefined) out[key] = source[key] + } + return out +} + +/** + * Replace selected fields' before/after values with a compact summary so large + * config blobs (dashboard widgets, query drafts) don't bloat the audit row. + */ +export const compactAuditChanges = ( + changes: AuditChanges | undefined, + summarize: Record unknown>, +): AuditChanges | undefined => { + if (changes === undefined) return undefined + const before: Record = { ...changes.before } + const after: Record = { ...changes.after } + for (const field of changes.fields) { + const summary = summarize[field] + if (summary === undefined) continue + if (field in before) before[field] = summary(before[field]) + if (field in after) after[field] = summary(after[field]) + } + return { fields: changes.fields, before, after } +} diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts new file mode 100644 index 000000000..a1a6e6e02 --- /dev/null +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -0,0 +1,141 @@ +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { AuditChanges, CurrentTenant } from "@maple/domain/http" +import { ActorId, ApiKeyId, UserId } from "@maple/domain/primitives" +import { + decodePublicId, + encodePublicId, + MapleApiV2, + paginateOffsetQuery, + PublicIdPrefixes, + timestamp, + V2ParameterInvalid, +} from "@maple/domain/http/v2" +import type { V2AuditLogEntry } from "@maple/domain/http/v2" +import type { AuditLogEntryRow } from "@maple/db" +import { Effect, Option, Schema } from "effect" +import { AuditLogService } from "@/services/audit/AuditLogService" +import type { AuditLogListFilters } from "@/services/audit/AuditLogService" + +const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) +const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) +const decodeUserIdOption = Schema.decodeUnknownOption(UserId) + +type ActorIdentityFilter = Pick + +/** + * Resolve the public `actor_id` filter to the column it identifies: `key_…` → + * the API key, `actor_…` → the agent, anything else → a (Clerk-issued, already + * public) user ID. + */ +const actorIdentityFilter = (publicActorId: string) => { + const invalid = V2ParameterInvalid.make("Invalid actor_id.", { param: "actor_id" }) + const succeed = (filter: ActorIdentityFilter) => Effect.succeed(filter) + const asApiKey = decodePublicId(PublicIdPrefixes.apiKey, publicActorId) + if (asApiKey !== null) { + return Option.match(decodeApiKeyIdOption(asApiKey), { + onNone: () => Effect.fail(invalid), + onSome: (apiKeyId) => succeed({ apiKeyId }), + }) + } + const asActor = decodePublicId(PublicIdPrefixes.actor, publicActorId) + if (asActor !== null) { + return Option.match(decodeActorIdOption(asActor), { + onNone: () => Effect.fail(invalid), + onSome: (actorId) => succeed({ actorId }), + }) + } + return Option.match(decodeUserIdOption(publicActorId), { + onNone: () => Effect.fail(invalid), + onSome: (userId) => succeed({ userId }), + }) +} + +const isJsonRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeChangesOption = Schema.decodeUnknownOption(AuditChanges) + +/** The actor's public identifier, matching the ID style of its own resource. */ +const publicActorId = (row: AuditLogEntryRow): string | null => { + switch (row.actorType) { + case "api_key": + return row.apiKeyId === null ? null : encodePublicId(PublicIdPrefixes.apiKey, row.apiKeyId) + case "agent": + return row.actorId === null ? null : encodePublicId(PublicIdPrefixes.actor, row.actorId) + case "user": + // Clerk user IDs are already prefixed public IDs — passed through as-is. + return row.userId + case "system": + return null + } +} + +const toV2AuditLogEntry = (row: AuditLogEntryRow): V2AuditLogEntry => ({ + id: row.id, + object: "audit_log_entry", + action: row.action, + outcome: row.outcome, + denial_reason: row.denialReason, + actor_type: row.actorType, + actor_id: publicActorId(row), + actor_name: row.actorLabel, + affected_user: row.affectedUserId, + source: row.source, + resource_type: row.resourceType, + resource_id: row.resourceId, + changes: Option.getOrNull(decodeChangesOption(row.changesJson)), + metadata: isJsonRecord(row.metadataJson) ? row.metadataJson : null, + request_id: row.requestId, + origin_ip: row.originIp, + origin_country: row.originCountry, + occurred_at: timestamp(row.occurredAt.toISOString()), + recorded_at: timestamp(row.recordedAt.toISOString()), +}) + +export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", (handlers) => + Effect.gen(function* () { + const audit = yield* AuditLogService + + return handlers.handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const identity = + query.actor_id !== undefined ? yield* actorIdentityFilter(query.actor_id) : undefined + const affectedUser = + query.affected_user !== undefined + ? yield* Option.match(decodeUserIdOption(query.affected_user), { + onNone: () => + Effect.fail( + V2ParameterInvalid.make("Invalid affected_user.", { param: "affected_user" }), + ), + onSome: (userId) => Effect.succeed(userId), + }) + : undefined + const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => + audit + .list(tenant.orgId, { + ...(query.actor_type !== undefined ? { actorType: query.actor_type } : undefined), + ...identity, + ...(affectedUser !== undefined ? { affectedUserId: affectedUser } : undefined), + ...(query.action !== undefined ? { action: query.action } : undefined), + ...(query.outcome !== undefined ? { outcome: query.outcome } : undefined), + ...(query.resource_type !== undefined + ? { resourceType: query.resource_type } + : undefined), + ...(query.resource_id !== undefined + ? { resourceId: query.resource_id } + : undefined), + ...(query.changed !== undefined ? { changedField: query.changed } : undefined), + ...(query.request_id !== undefined ? { requestId: query.request_id } : undefined), + ...(query.since !== undefined ? { sinceMs: Date.parse(query.since) } : undefined), + ...(query.until !== undefined ? { untilMs: Date.parse(query.until) } : undefined), + limit, + offset, + }) + .pipe(Effect.map((rows) => rows.map(toV2AuditLogEntry))), + ) + return { object: "list" as const, ...page } + }), + ) + }), +) diff --git a/apps/api/src/routes/v2/config-resources.http.test.ts b/apps/api/src/routes/v2/config-resources.http.test.ts index dcdd10091..e5d3f47a2 100644 --- a/apps/api/src/routes/v2/config-resources.http.test.ts +++ b/apps/api/src/routes/v2/config-resources.http.test.ts @@ -9,6 +9,7 @@ import type { WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQue import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { Env } from "@/platform/Env" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -115,6 +116,7 @@ const makeHarness = () => { // session_replays (in AllV2GroupLayersLive) needs the warehouse at the routes level. Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.test.ts b/apps/api/src/routes/v2/dashboards.http.test.ts index 025d06de8..1418b1fa3 100644 --- a/apps/api/src/routes/v2/dashboards.http.test.ts +++ b/apps/api/src/routes/v2/dashboards.http.test.ts @@ -11,6 +11,7 @@ import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { V2TransportErrorBoundaryLive } from "./error-envelope" import { AlertsServiceStubLayer, @@ -65,6 +66,7 @@ const makeHarness = () => { Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.ts b/apps/api/src/routes/v2/dashboards.http.ts index efc99098a..604285f25 100644 --- a/apps/api/src/routes/v2/dashboards.http.ts +++ b/apps/api/src/routes/v2/dashboards.http.ts @@ -9,9 +9,11 @@ import { PortableDashboardDocument, } from "@maple/domain/http" import { + encodePublicId, MapleApiV2, LIST_LIMIT_DEFAULT, paginateArray, + PublicIdPrefixes, V2ParameterInvalid, V2ParameterMissing, } from "@maple/domain/http/v2" @@ -30,6 +32,8 @@ import type { DashboardId } from "@maple/domain/primitives" import { Clock, Effect, Option, Schema } from "effect" import { getTemplateById, listTemplateMetadata } from "@/dashboard-templates" import type { TemplateParameterValues } from "@/dashboard-templates" +import { compactAuditChanges, diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { convertPersesDashboardToPortable } from "@/services/dashboards/perses-dashboard-import" @@ -174,6 +178,22 @@ const applyUpdate = ( }) } +/** Update-payload fields diffable through the wire shape; layout blobs get summarized. */ +const dashboardAuditKeys: ReadonlyArray = [ + "name", + "description", + "tags", + "timeRange", + "widgets", + "sections", + "variables", + "refreshIntervalSeconds", +] + +/** Layout arrays are config blobs — audit their size, not their bodies. */ +const summarizeListBlob = (label: string) => (value: unknown) => + Array.isArray(value) ? `<${value.length} ${label}>` : "" + const encodeVersionCursor = (versionNumber: number): string => `ver_${versionNumber.toString(36)}` const decodeVersionCursor = (cursor: string): number | null => { @@ -270,6 +290,17 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards "maple.share.id": created.id, mode: created.mode, }) + yield* recordHttpAudit("dashboard_share.created", { + resourceType: "dashboard_share", + resourceId: encodePublicId(PublicIdPrefixes.dashboardShare, created.id), + metadata: { + mode: created.mode, + dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, context.scope.dashboardId), + ...(context.scope.widgetId === null + ? undefined + : { widget_id: context.scope.widgetId }), + }, + }) return toV2DashboardShare(created) }) @@ -298,6 +329,15 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards ) yield* logShare("dashboard share revoked", context, { hadLiveShare: tombstone.revoked }) + if (tombstone.revoked) { + yield* recordHttpAudit("dashboard_share.deleted", { + resourceType: "dashboard_share", + metadata: { + dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, dashboardId), + ...(widgetId === null ? undefined : { widget_id: widgetId }), + }, + }) + } // `deleted: true` regardless of whether a live share existed: "stop // sharing" is a statement about the end state, and the dialog must be @@ -336,6 +376,11 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards tenant.userId, toPortable(payload), ) + yield* recordHttpAudit("dashboard.created", { + resourceType: "dashboard", + resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + metadata: { name: dashboard.name }, + }) return toV2DashboardMutation(dashboard) }), @@ -346,12 +391,37 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards const updatedAt = asIsoDateTime( new Date(yield* Clock.currentTimeMillis).toISOString(), ) + // Capture the pre-state the mutate callback already reads, for the diff. + let previous: DashboardDocument | undefined const dashboard = yield* persistence.mutate( tenant.orgId, tenant.userId, params.id, - (current) => Effect.succeed(applyUpdate(current, payload, updatedAt)), + (current) => { + previous = current + return Effect.succeed(applyUpdate(current, payload, updatedAt)) + }, ) + const changes = + previous === undefined + ? undefined + : compactAuditChanges( + diffAuditChanges( + pickPresentFields(dashboardAuditKeys, payload, toV2Dashboard(previous)), + pickPresentFields(dashboardAuditKeys, payload, toV2Dashboard(dashboard)), + ), + { + widgets: summarizeListBlob("widgets"), + sections: summarizeListBlob("sections"), + variables: summarizeListBlob("variables"), + }, + ) + yield* recordHttpAudit("dashboard.updated", { + resourceType: "dashboard", + resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + ...(changes !== undefined ? { changes } : undefined), + metadata: { name: dashboard.name }, + }) return toV2DashboardMutation(dashboard) }), @@ -360,6 +430,10 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* persistence.delete(tenant.orgId, params.id) + yield* recordHttpAudit("dashboard.deleted", { + resourceType: "dashboard", + resourceId: encodePublicId(PublicIdPrefixes.dashboard, deleted.id), + }) return { id: deleted.id, diff --git a/apps/api/src/routes/v2/ingest-keys.http.ts b/apps/api/src/routes/v2/ingest-keys.http.ts index cddf23be2..b337fd79c 100644 --- a/apps/api/src/routes/v2/ingest-keys.http.ts +++ b/apps/api/src/routes/v2/ingest-keys.http.ts @@ -4,6 +4,7 @@ import { CurrentTenant } from "@maple/domain/http" import { MapleApiV2, V2InsufficientPermissions } from "@maple/domain/http/v2" import type { V2IngestKeys } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { OrgIngestKeysService } from "@/services/org/OrgIngestKeysService" import { requireAdmin } from "@/services/auth/auth" @@ -37,6 +38,10 @@ export const HttpV2IngestKeysLive = HttpApiBuilder.group(MapleApiV2, "ingestKeys const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly("roll")) const keys = yield* ingestKeys.rerollPublic(tenant.orgId, tenant.userId) + yield* recordHttpAudit("ingest_key.rolled", { + resourceType: "ingest_key", + metadata: { key_type: "public" }, + }) return toV2IngestKeys(keys) }), @@ -46,6 +51,10 @@ export const HttpV2IngestKeysLive = HttpApiBuilder.group(MapleApiV2, "ingestKeys const tenant = yield* CurrentTenant.Context yield* requireAdmin(tenant.roles, adminOnly("roll")) const keys = yield* ingestKeys.rerollPrivate(tenant.orgId, tenant.userId) + yield* recordHttpAudit("ingest_key.rolled", { + resourceType: "ingest_key", + metadata: { key_type: "private" }, + }) return toV2IngestKeys(keys) }), diff --git a/apps/api/src/routes/v2/integrations.http.test.ts b/apps/api/src/routes/v2/integrations.http.test.ts index a21d9b9b0..ea54c4b3a 100644 --- a/apps/api/src/routes/v2/integrations.http.test.ts +++ b/apps/api/src/routes/v2/integrations.http.test.ts @@ -23,6 +23,7 @@ import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { SLACK_CALLBACK_PATH, SlackIntegrationService, @@ -170,6 +171,7 @@ const makeHarness = (slack: Partial = {}, planetscal Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/mobile-devices.http.test.ts b/apps/api/src/routes/v2/mobile-devices.http.test.ts index 416e0826a..c88e67c07 100644 --- a/apps/api/src/routes/v2/mobile-devices.http.test.ts +++ b/apps/api/src/routes/v2/mobile-devices.http.test.ts @@ -7,6 +7,7 @@ import { MapleApiV2, encodePublicId } from "@maple/domain/http/v2" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { Env } from "@/platform/Env" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -79,6 +80,7 @@ const makeHarness = () => { Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/phase1-resources.http.test.ts b/apps/api/src/routes/v2/phase1-resources.http.test.ts index f273bce19..fa9a51b64 100644 --- a/apps/api/src/routes/v2/phase1-resources.http.test.ts +++ b/apps/api/src/routes/v2/phase1-resources.http.test.ts @@ -50,6 +50,7 @@ import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryServic import { Env } from "@/platform/Env" import { AnomalyDetectionService } from "@/services/alerts/AnomalyDetectionService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -572,6 +573,7 @@ const makeHarness = ( Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/scrape-targets.http.ts b/apps/api/src/routes/v2/scrape-targets.http.ts index e8916bd5c..0c9d1651b 100644 --- a/apps/api/src/routes/v2/scrape-targets.http.ts +++ b/apps/api/src/routes/v2/scrape-targets.http.ts @@ -1,9 +1,18 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import type { ScrapeTargetResponse } from "@maple/domain/http" import { CreateScrapeTargetRequest, CurrentTenant, UpdateScrapeTargetRequest } from "@maple/domain/http" -import { MapleApiV2, paginateArray, paginateOffsetQuery, timestamp } from "@maple/domain/http/v2" +import { + encodePublicId, + MapleApiV2, + paginateArray, + paginateOffsetQuery, + PublicIdPrefixes, + timestamp, +} from "@maple/domain/http/v2" import type { V2ScrapeTarget, V2ScrapeTargetCheck } from "@maple/domain/http/v2" import { Effect } from "effect" +import { diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ScrapeTargetsService } from "@/services/integrations/ScrapeTargetsService" const toV2ScrapeTarget = (target: ScrapeTargetResponse): V2ScrapeTarget => ({ @@ -28,6 +37,31 @@ const toV2ScrapeTarget = (target: ScrapeTargetResponse): V2ScrapeTarget => ({ updated_at: target.updatedAt, }) +/** Update-payload fields diffable through the wire shape; credentials never appear. */ +const targetAuditKeys: ReadonlyArray< + | "name" + | "url" + | "organization" + | "include_branches" + | "exclude_branches" + | "scrape_interval_seconds" + | "labels_json" + | "auth_type" + | "service_name" + | "enabled" +> = [ + "name", + "url", + "organization", + "include_branches", + "exclude_branches", + "scrape_interval_seconds", + "labels_json", + "auth_type", + "service_name", + "enabled", +] + export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeTargets", (handlers) => Effect.gen(function* () { const service = yield* ScrapeTargetsService @@ -96,12 +130,19 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT }), ) + yield* recordHttpAudit("scrape_target.created", { + resourceType: "scrape_target", + resourceId: encodePublicId(PublicIdPrefixes.scrapeTarget, created.id), + metadata: { name: created.name }, + }) + return toV2ScrapeTarget(created) }), ) .handle("update", ({ params, payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + const current = yield* service.get(tenant.orgId, params.id) const updated = yield* service.update( tenant.orgId, params.id, @@ -144,6 +185,26 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT }), ) + const observable = diffAuditChanges( + pickPresentFields(targetAuditKeys, payload, toV2ScrapeTarget(current)), + pickPresentFields(targetAuditKeys, payload, toV2ScrapeTarget(updated)), + ) + // Credentials are write-only: audit that they rotated, never their value. + const changes = + payload.auth_credentials !== undefined + ? { + fields: [...(observable?.fields ?? []), "auth_credentials"], + before: { ...observable?.before, auth_credentials: "" }, + after: { ...observable?.after, auth_credentials: "" }, + } + : observable + yield* recordHttpAudit("scrape_target.updated", { + resourceType: "scrape_target", + resourceId: encodePublicId(PublicIdPrefixes.scrapeTarget, updated.id), + ...(changes !== undefined ? { changes } : undefined), + metadata: { name: updated.name }, + }) + return toV2ScrapeTarget(updated) }), ) @@ -151,6 +212,10 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* service.delete(tenant.orgId, params.id) + yield* recordHttpAudit("scrape_target.deleted", { + resourceType: "scrape_target", + resourceId: encodePublicId(PublicIdPrefixes.scrapeTarget, deleted.id), + }) return { id: deleted.id, object: "scrape_target" as const, deleted: true as const } }), diff --git a/apps/api/src/routes/v2/setup-audit.http.test.ts b/apps/api/src/routes/v2/setup-audit.http.test.ts index 79698faa6..ad678e8a4 100644 --- a/apps/api/src/routes/v2/setup-audit.http.test.ts +++ b/apps/api/src/routes/v2/setup-audit.http.test.ts @@ -10,6 +10,7 @@ import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryServic import { Database } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -142,6 +143,7 @@ const makeHarness = (warehouse: WarehouseQueryServiceApi = warehouseStub()) => { Layer.provide(TelemetryServiceStubsLayer), Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts index 90d158721..868835b9b 100644 --- a/apps/api/src/routes/v2/telemetry.http.test.ts +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -12,6 +12,7 @@ import { type WarehouseQueryServiceApi, } from "@/services/warehouse/WarehouseQueryService" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -275,6 +276,7 @@ const makeHarness = ( Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 6ad42815e..43b548a6d 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -41,6 +41,8 @@ import { HttpV2InvestigationsLive } from "./investigations.http" import { HttpV2MobileDevicesLive } from "./mobile-devices.http" import { HttpV2OrganizationLive } from "./organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./recommendations.http" +import { HttpV2AuditLogLive } from "./audit-log.http" +import { AuditLogService } from "@/services/audit/AuditLogService" import { HttpV2ScrapeTargetsLive } from "./scrape-targets.http" import { HttpV2SessionReplaysLive } from "./session-replays.http" import { HttpV2InstrumentationAuditLive } from "./setup-audit.http" @@ -76,6 +78,8 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2IngestKeysLive, HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, + // Real service, no stub: it needs only the Database every harness already provides. + HttpV2AuditLogLive.pipe(Layer.provide(AuditLogService.layer)), HttpV2ScrapeTargetsLive, HttpV2InstrumentationRecommendationsLive, HttpV2InstrumentationAuditLive, @@ -118,6 +122,10 @@ export const AllV2GroupLayersLive = Layer.mergeAll( }), ), ), +).pipe( + // Mutation handlers across the groups record audit entries; the real service + // needs only the Database every harness already provides. + Layer.provide(AuditLogService.layer), ) export const ApiV2RateLimiterAllowAllLayer = Layer.succeed(ApiV2RateLimiter, { diff --git a/apps/api/src/routes/v2/widget-credentials.http.test.ts b/apps/api/src/routes/v2/widget-credentials.http.test.ts index 05b784def..c09ec82be 100644 --- a/apps/api/src/routes/v2/widget-credentials.http.test.ts +++ b/apps/api/src/routes/v2/widget-credentials.http.test.ts @@ -7,6 +7,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -77,6 +78,7 @@ const makeHarness = () => { Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/widget-summary.http.test.ts b/apps/api/src/routes/v2/widget-summary.http.test.ts index 6113aeb70..d7ca174f3 100644 --- a/apps/api/src/routes/v2/widget-summary.http.test.ts +++ b/apps/api/src/routes/v2/widget-summary.http.test.ts @@ -15,6 +15,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { ApiAuthorizationV2Layer } from "@/services/auth/ApiAuthorizationV2Layer" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" @@ -190,6 +191,7 @@ const makeHarness = (options: { Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), + Layer.provideMerge(AuditLogService.layer), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/runtime/graph-boundaries.test.ts b/apps/api/src/runtime/graph-boundaries.test.ts index cadd8f9eb..70f3a3051 100644 --- a/apps/api/src/runtime/graph-boundaries.test.ts +++ b/apps/api/src/runtime/graph-boundaries.test.ts @@ -71,6 +71,8 @@ describe("API runtime graph boundaries", () => { "AlertReadModelsServiceLive", "AlertRulesServiceLive", "AlertsServiceLive", + // Lets `register_agent` (and issue-workflow mutations) write org audit entries. + "AuditLogService.layer", "DashboardPersistenceService.layer", "ErrorActorsServiceLive", "ErrorIssueReadModelsServiceLive", diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 40adb9d09..d8dd32b35 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -49,6 +49,8 @@ import { HttpV2InvestigationsLive } from "@/routes/v2/investigations.http" import { HttpV2MobileDevicesLive } from "@/routes/v2/mobile-devices.http" import { HttpV2OrganizationLive } from "@/routes/v2/organization.http" import { HttpV2InstrumentationRecommendationsLive } from "@/routes/v2/recommendations.http" +import { HttpV2AuditLogLive } from "@/routes/v2/audit-log.http" +import { AuditLogService } from "@/services/audit/AuditLogService" import { HttpV2ScrapeTargetsLive } from "@/routes/v2/scrape-targets.http" import { HttpV2InstrumentationAuditLive } from "@/routes/v2/setup-audit.http" import { HttpV2SessionReplaysLive } from "@/routes/v2/session-replays.http" @@ -130,6 +132,7 @@ const ApiV2Routes = HttpApiBuilder.layer(MapleApiV2).pipe( HttpV2PlanetScaleIntegrationsLive, HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, + HttpV2AuditLogLive, HttpV2ScrapeTargetsLive, HttpV2InstrumentationRecommendationsLive, HttpV2InstrumentationAuditLive, @@ -184,6 +187,8 @@ export const ApiAuthLive = Layer.mergeAll( ).pipe( Layer.provideMerge(ApiV2RateLimiter.layer), Layer.provideMerge(ApiKeysService.layer), + // Denied attempts are audited from inside the auth layers themselves. + Layer.provideMerge(AuditLogService.layer), // Membership verification for `x-maple-org-id`. Only the v2 layer asks for // it; without it that layer cannot build, which is deliberate — the header // must never end up silently ignored in a runtime that forgot to wire this. diff --git a/apps/api/src/runtime/mcp-service-graph.ts b/apps/api/src/runtime/mcp-service-graph.ts index 883798415..4adf4dda6 100644 --- a/apps/api/src/runtime/mcp-service-graph.ts +++ b/apps/api/src/runtime/mcp-service-graph.ts @@ -2,6 +2,7 @@ import { EdgeCacheService } from "@maple/cache" import { BucketCacheService } from "@maple/query-engine/caching" import { Layer } from "effect" import { McpToolExecutor } from "@/mcp/dispatcher" +import { AuditLogService } from "@/services/audit/AuditLogService" import { CacheBackendLive } from "@/platform/CacheBackendLive" import { EmailService } from "@/platform/EmailService" import { Env } from "@/platform/Env" @@ -104,7 +105,7 @@ const NotificationDispatcherLive = NotificationDispatcher.layer.pipe( const ErrorActorsServiceLive = ErrorActorsService.layer const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(ErrorActorsServiceLive), + Layer.provide(Layer.mergeAll(ErrorActorsServiceLive, AuditLogService.layer)), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer const ErrorIssueReadModelsServiceLive = ErrorIssueReadModelsService.layer.pipe( @@ -165,6 +166,7 @@ const McpRuntimeServicesLive = Layer.mergeAll( AlertReadModelsServiceLive, AlertRulesServiceLive, AlertsServiceLive, + AuditLogService.layer, DashboardPersistenceService.layer, ErrorActorsServiceLive, ErrorIssueReadModelsServiceLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 3c2a278de..8f263e7d4 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -53,6 +53,7 @@ import { GithubConnectService } from "@/services/integrations/vcs/vendor/github/ import { GithubHttp } from "@/services/integrations/vcs/vendor/github/GithubHttp" import { GithubProvider } from "@/services/integrations/vcs/vendor/github/GithubProvider" import { ApiKeysService } from "@/services/org/ApiKeysService" +import { AuditLogService } from "@/services/audit/AuditLogService" import { DemoService } from "@/services/org/DemoService" import { IngestAttributeMappingService } from "@/services/org/IngestAttributeMappingService" import { OnboardingService } from "@/services/org/OnboardingService" @@ -84,6 +85,7 @@ const EdgeCacheServiceLive = EdgeCacheService.layer.pipe(Layer.provide(CacheBack const CoreServicesLive = Layer.mergeAll( AuthService.layer, ApiKeysService.layer, + AuditLogService.layer, CliDeviceAuthService.layer, McpOAuthService.layer, CloudflareOAuthService.layer, @@ -176,6 +178,7 @@ const NotificationDispatcherLive = NotificationDispatcher.layer.pipe( const ErrorActorsServiceLive = ErrorActorsService.layer const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layer), Layer.provideMerge(ErrorActorsServiceLive), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer diff --git a/apps/api/src/services/audit/AuditLogService.test.ts b/apps/api/src/services/audit/AuditLogService.test.ts new file mode 100644 index 000000000..267d965a4 --- /dev/null +++ b/apps/api/src/services/audit/AuditLogService.test.ts @@ -0,0 +1,202 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { OrgId, UserId } from "@maple/domain/primitives" +import { Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "./AuditLogService" + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asUserId = Schema.decodeUnknownSync(UserId) + +const ORG = asOrgId("org_audit_log_test") +const USER = asUserId("user_audit_log_test") +const createdDbs: TestDb[] = [] + +afterEach(() => cleanupTestDbs(createdDbs)) + +const makeLayer = () => AuditLogService.layer.pipe(Layer.provide(createTestDb(createdDbs).layer)) + +/** Three entries with distinct timestamps: user, then api_key, then agent. */ +const seedThree = Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + resourceType: "dashboard", + resourceId: "dash_first", + metadata: { name: "First" }, + }) + yield* TestClock.adjust("1 second") + yield* audit.record({ + orgId: ORG, + actor: { type: "api_key" }, + source: "api", + action: "alert_rule.updated", + }) + yield* TestClock.adjust("1 second") + yield* audit.record({ + orgId: ORG, + actor: { type: "agent", label: "triage-bot" }, + source: "mcp", + action: "error_issue.state_change", + }) +}) + +describe("AuditLogService", () => { + it.effect("round-trips a recorded entry and lists newest first", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + + const rows = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(rows.map((row) => row.action)).toEqual([ + "error_issue.state_change", + "alert_rule.updated", + "dashboard.created", + ]) + + const oldest = rows[2]! + expect(oldest.actorType).toBe("user") + expect(oldest.userId).toBe(USER) + expect(oldest.source).toBe("dashboard") + expect(oldest.resourceType).toBe("dashboard") + expect(oldest.resourceId).toBe("dash_first") + expect(oldest.metadataJson).toEqual({ name: "First" }) + + const newest = rows[0]! + expect(newest.actorType).toBe("agent") + expect(newest.actorLabel).toBe("triage-bot") + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("filters by actor type", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + + const apiKeyRows = yield* audit.list(ORG, { actorType: "api_key", limit: 10, offset: 0 }) + expect(apiKeyRows.map((row) => row.action)).toEqual(["alert_rule.updated"]) + + const systemRows = yield* audit.list(ORG, { actorType: "system", limit: 10, offset: 0 }) + expect(systemRows).toEqual([]) + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("pages with offset and limit in newest-first order", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + + const firstPage = yield* audit.list(ORG, { limit: 2, offset: 0 }) + expect(firstPage.map((row) => row.action)).toEqual([ + "error_issue.state_change", + "alert_rule.updated", + ]) + + const secondPage = yield* audit.list(ORG, { limit: 2, offset: 2 }) + expect(secondPage.map((row) => row.action)).toEqual(["dashboard.created"]) + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("records denied outcomes and filters by outcome", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "alert_rule.delete", + outcome: "denied", + denialReason: "missing role: admin", + }) + + const denied = yield* audit.list(ORG, { outcome: "denied", limit: 10, offset: 0 }) + expect(denied.map((row) => row.action)).toEqual(["alert_rule.delete"]) + expect(denied[0]!.outcome).toBe("denied") + expect(denied[0]!.denialReason).toBe("missing role: admin") + + const allowed = yield* audit.list(ORG, { outcome: "allowed", limit: 10, offset: 0 }) + expect(allowed).toHaveLength(3) + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("stores update diffs and filters by changed field", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* seedThree + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.updated", + changes: { fields: ["name"], before: { name: "a" }, after: { name: "b" } }, + }) + + const rows = yield* audit.list(ORG, { changedField: "name", limit: 10, offset: 0 }) + expect(rows.map((row) => row.action)).toEqual(["dashboard.updated"]) + expect(rows[0]!.changedFields).toEqual(["name"]) + expect(rows[0]!.changesJson).toEqual({ + fields: ["name"], + before: { name: "a" }, + after: { name: "b" }, + }) + + const none = yield* audit.list(ORG, { changedField: "description", limit: 10, offset: 0 }) + expect(none).toEqual([]) + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("publishes to the audit queue instead of writing when the binding is present", () => { + const sent: unknown[] = [] + return Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + }) + + expect(sent).toHaveLength(1) + expect(sent[0]).toMatchObject({ orgId: ORG, action: "dashboard.created" }) + // The consumer performs the insert; nothing lands in the DB directly. + const rows = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(rows).toEqual([]) + }).pipe( + Effect.provide( + makeLayer().pipe( + Layer.provide( + Layer.succeed(WorkerEnvironment, { + AUDIT_EVENTS_QUEUE: { + send: async (message: unknown) => { + sent.push(message) + }, + }, + }), + ), + ), + ), + ) + }) + + it.effect("writes directly when the queue binding is absent from the worker environment", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + }) + + const rows = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(rows.map((row) => row.action)).toEqual(["dashboard.created"]) + }).pipe( + Effect.provide(makeLayer().pipe(Layer.provide(Layer.succeed(WorkerEnvironment, {})))), + ), + ) +}) diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts new file mode 100644 index 000000000..4472dcb68 --- /dev/null +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -0,0 +1,291 @@ +import { randomUUID } from "node:crypto" +import { HttpServerRequest } from "effect/unstable/http" +import { AuditLogPersistenceError, CurrentTenant } from "@maple/domain/http" +import type { AuditActorType, AuditChanges, AuditLogSource, AuditOutcome } from "@maple/domain/http" +import type { ActorId, ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import { AuditLogEntryId as AuditLogEntryIdSchema } from "@maple/domain/primitives" +import { auditLogEntries, type AuditLogEntryRow } from "@maple/db" +import { and, arrayContains, desc, eq, gte, lte } from "drizzle-orm" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" +import type { Queue } from "@cloudflare/workers-types" +import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { Database } from "@/platform/DatabaseLive" +import { msToDate } from "@/platform/time" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogEvent, auditEventToInsert, encodeAuditLogEventSync } from "./audit-event" + +const decodeAuditLogEntryIdSync = Schema.decodeUnknownSync(AuditLogEntryIdSchema) + +/** Producer binding name; the paired `*_NAME` var drives consumer dispatch. */ +export const AUDIT_EVENTS_QUEUE_BINDING = "AUDIT_EVENTS_QUEUE" + +const toPersistenceError = (error: unknown) => + new AuditLogPersistenceError({ + message: error instanceof Error ? error.message : "Audit log query failed", + }) + +/** The credential-holder behind an audited action, as known at the call site. */ +export interface AuditActorRef { + readonly type: AuditActorType + readonly userId?: UserId + readonly apiKeyId?: ApiKeyId + readonly actorId?: ActorId + readonly label?: string +} + +export interface AuditLogRecordInput { + readonly orgId: OrgId + readonly actor: AuditActorRef + readonly source: AuditLogSource + /** `.`, e.g. `alert_rule.created`. */ + readonly action: string + /** Defaults to `"allowed"`; denied attempts pass `"denied"` + `denialReason`. */ + readonly outcome?: AuditOutcome + readonly denialReason?: string + readonly affectedUserId?: UserId + readonly resourceType?: string + readonly resourceId?: string + readonly changes?: AuditChanges + readonly metadata?: Record + readonly requestId?: string + readonly originIp?: string + readonly originCountry?: string +} + +export interface AuditLogListFilters { + readonly actorType?: AuditActorType + /** At most one of the three actor-identity filters is set per request. */ + readonly userId?: UserId + readonly apiKeyId?: ApiKeyId + readonly actorId?: ActorId + readonly affectedUserId?: UserId + readonly action?: string + readonly outcome?: AuditOutcome + readonly resourceType?: string + /** Matches the stored public form (e.g. `dash_…`). */ + readonly resourceId?: string + /** Field name that an update's diff must have touched. */ + readonly changedField?: string + readonly requestId?: string + readonly sinceMs?: number + readonly untilMs?: number + readonly limit: number + readonly offset: number +} + +export interface AuditLogServiceApi { + /** + * Append one entry, durably: published to the audit events queue when the + * binding is present (the consumer performs the insert, retried by the + * queue), written straight to Postgres otherwise (tests, local dev, crons). + * Never fails: a mutation that succeeded must not 500 because its audit + * write did not — terminal failures are logged and swallowed. + */ + readonly record: (input: AuditLogRecordInput) => Effect.Effect + readonly list: ( + orgId: OrgId, + filters: AuditLogListFilters, + ) => Effect.Effect, AuditLogPersistenceError> +} + +export class AuditLogService extends Context.Service()( + "@maple/api/services/AuditLogService", + { + make: Effect.gen(function* () { + const database = yield* Database + // Optional so PGlite tests and non-Worker runtimes fall back to direct + // writes without providing a WorkerEnvironment. + const workerEnv = yield* Effect.serviceOption(WorkerEnvironment) + const queue = Option.match(workerEnv, { + onNone: () => undefined, + onSome: (env) => { + const binding = env[AUDIT_EVENTS_QUEUE_BINDING] + // SAFETY: the binding slot is owned by this service; anything present is the queue. + return binding === undefined ? undefined : (binding as Queue) + }, + }) + + const insertDirect = (event: AuditLogEvent) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + yield* database.execute((db) => + db.insert(auditLogEntries).values(auditEventToInsert(event, now)).onConflictDoNothing(), + ) + }) + + const publish = (event: AuditLogEvent) => + queue === undefined + ? insertDirect(event) + : Effect.tryPromise({ + try: () => queue.send(encodeAuditLogEventSync(event)), + catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), + }).pipe( + // Queue unavailability must not lose the entry: degrade to a + // direct write before giving up. + Effect.catchCause((cause) => + Effect.logWarning("Audit queue send failed; writing directly", { cause }).pipe( + Effect.andThen(insertDirect(event)), + ), + ), + ) + + const record: AuditLogServiceApi["record"] = Effect.fn("AuditLogService.record")(function* ( + input, + ) { + const now = yield* Clock.currentTimeMillis + const event = new AuditLogEvent({ + orgId: input.orgId, + id: decodeAuditLogEntryIdSync(randomUUID()), + actorType: input.actor.type, + ...(input.actor.userId !== undefined ? { userId: input.actor.userId } : undefined), + ...(input.actor.apiKeyId !== undefined ? { apiKeyId: input.actor.apiKeyId } : undefined), + ...(input.actor.actorId !== undefined ? { actorId: input.actor.actorId } : undefined), + ...(input.actor.label !== undefined ? { actorLabel: input.actor.label } : undefined), + ...(input.affectedUserId !== undefined + ? { affectedUserId: input.affectedUserId } + : undefined), + source: input.source, + action: input.action, + outcome: input.outcome ?? "allowed", + ...(input.denialReason !== undefined ? { denialReason: input.denialReason } : undefined), + ...(input.resourceType !== undefined ? { resourceType: input.resourceType } : undefined), + ...(input.resourceId !== undefined ? { resourceId: input.resourceId } : undefined), + ...(input.changes !== undefined ? { changes: input.changes } : undefined), + ...(input.metadata !== undefined ? { metadata: input.metadata } : undefined), + ...(input.requestId !== undefined ? { requestId: input.requestId } : undefined), + ...(input.originIp !== undefined ? { originIp: input.originIp } : undefined), + ...(input.originCountry !== undefined + ? { originCountry: input.originCountry } + : undefined), + occurredAtMs: now, + }) + // High-signal by definition — surfaced as a warning so Maple's own + // error/log alerting can watch for spikes of refused attempts. + if (event.outcome === "denied") { + yield* Effect.logWarning("Audit: denied action").pipe( + Effect.annotateLogs({ + orgId: event.orgId, + action: event.action, + actorType: event.actorType, + denialReason: event.denialReason ?? "", + }), + ) + } + yield* publish(event).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Audit log write failed", { action: input.action, cause }), + ), + ) + }) + + const list: AuditLogServiceApi["list"] = Effect.fn("AuditLogService.list")(function* ( + orgId, + filters, + ) { + const conditions = [ + eq(auditLogEntries.orgId, orgId), + ...(filters.actorType !== undefined + ? [eq(auditLogEntries.actorType, filters.actorType)] + : []), + ...(filters.userId !== undefined ? [eq(auditLogEntries.userId, filters.userId)] : []), + ...(filters.apiKeyId !== undefined + ? [eq(auditLogEntries.apiKeyId, filters.apiKeyId)] + : []), + ...(filters.actorId !== undefined ? [eq(auditLogEntries.actorId, filters.actorId)] : []), + ...(filters.affectedUserId !== undefined + ? [eq(auditLogEntries.affectedUserId, filters.affectedUserId)] + : []), + ...(filters.action !== undefined ? [eq(auditLogEntries.action, filters.action)] : []), + ...(filters.outcome !== undefined ? [eq(auditLogEntries.outcome, filters.outcome)] : []), + ...(filters.resourceType !== undefined + ? [eq(auditLogEntries.resourceType, filters.resourceType)] + : []), + ...(filters.resourceId !== undefined + ? [eq(auditLogEntries.resourceId, filters.resourceId)] + : []), + ...(filters.changedField !== undefined + ? [arrayContains(auditLogEntries.changedFields, [filters.changedField])] + : []), + ...(filters.requestId !== undefined + ? [eq(auditLogEntries.requestId, filters.requestId)] + : []), + ...(filters.sinceMs !== undefined + ? [gte(auditLogEntries.occurredAt, msToDate(filters.sinceMs))] + : []), + ...(filters.untilMs !== undefined + ? [lte(auditLogEntries.occurredAt, msToDate(filters.untilMs))] + : []), + ] + return yield* database + .execute((db) => + db + .select() + .from(auditLogEntries) + .where(and(...conditions)) + .orderBy(desc(auditLogEntries.occurredAt), desc(auditLogEntries.id)) + .limit(filters.limit) + .offset(filters.offset), + ) + .pipe(Effect.mapError(toPersistenceError)) + }) + + return { record, list } + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} + +/** Request forensics for an audit entry, read off the Cloudflare request headers. */ +const requestContext = Effect.gen(function* () { + const request = yield* Effect.serviceOption(HttpServerRequest.HttpServerRequest) + return Option.match(request, { + onNone: () => ({}), + onSome: (req) => ({ + ...(req.headers["cf-ray"] !== undefined ? { requestId: req.headers["cf-ray"] } : undefined), + ...(req.headers["cf-connecting-ip"] !== undefined + ? { originIp: req.headers["cf-connecting-ip"] } + : undefined), + ...(req.headers["cf-ipcountry"] !== undefined + ? { originCountry: req.headers["cf-ipcountry"] } + : undefined), + }), + }) +}) + +/** + * Record an audit entry for the current authenticated HTTP request, deriving + * the actor from the tenant plus the auth middleware's `CurrentAuditActor`, + * and request forensics (request id, origin) from the Cloudflare headers. + * Session requests (and requests that bypassed the standard middlewares) + * attribute to the user; API-key requests attribute to the key. + */ +export const recordHttpAudit = ( + action: string, + opts?: { + readonly resourceType?: string + readonly resourceId?: string + readonly changes?: AuditChanges + readonly affectedUserId?: UserId + readonly metadata?: Record + }, +) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const tenant = yield* CurrentTenant.Context + const info = yield* CurrentAuditActor + const context = yield* requestContext + const isApiKey = info?.type === "api_key" + yield* audit.record({ + orgId: tenant.orgId, + actor: { + type: isApiKey ? "api_key" : "user", + userId: tenant.userId, + ...(isApiKey && info.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), + }, + source: isApiKey ? "api" : "dashboard", + action, + ...context, + ...opts, + }) + }) diff --git a/apps/api/src/services/audit/audit-event.ts b/apps/api/src/services/audit/audit-event.ts new file mode 100644 index 000000000..30e79b5c5 --- /dev/null +++ b/apps/api/src/services/audit/audit-event.ts @@ -0,0 +1,62 @@ +import { AuditActorType, AuditChanges, AuditLogSource, AuditOutcome } from "@maple/domain/http" +import { ActorId, ApiKeyId, AuditLogEntryId, OrgId, UserId } from "@maple/domain/primitives" +import type { AuditLogEntryInsert } from "@maple/db" +import { Schema } from "effect" +import { msToDate } from "@/platform/time" + +/** + * The serialized audit event as it travels the audit queue. `occurredAtMs` is + * stamped by the producer; `recordedAt` exists only on the table row, stamped + * by whichever writer performs the insert. + */ +export class AuditLogEvent extends Schema.Class("AuditLogEvent")({ + orgId: OrgId, + id: AuditLogEntryId, + actorType: AuditActorType, + userId: Schema.optionalKey(UserId), + apiKeyId: Schema.optionalKey(ApiKeyId), + actorId: Schema.optionalKey(ActorId), + actorLabel: Schema.optionalKey(Schema.String), + affectedUserId: Schema.optionalKey(UserId), + source: AuditLogSource, + action: Schema.String, + outcome: AuditOutcome, + denialReason: Schema.optionalKey(Schema.String), + resourceType: Schema.optionalKey(Schema.String), + resourceId: Schema.optionalKey(Schema.String), + changes: Schema.optionalKey(AuditChanges), + metadata: Schema.optionalKey(Schema.Record(Schema.String, Schema.Unknown)), + requestId: Schema.optionalKey(Schema.String), + originIp: Schema.optionalKey(Schema.String), + originCountry: Schema.optionalKey(Schema.String), + occurredAtMs: Schema.Number, +}) {} + +export const decodeAuditLogEvent = Schema.decodeUnknownEffect(AuditLogEvent) +export const encodeAuditLogEventSync = Schema.encodeSync(AuditLogEvent) + +/** Lower a queue event to its table row; `recordedAtMs` is the insert time. */ +export const auditEventToInsert = (event: AuditLogEvent, recordedAtMs: number): AuditLogEntryInsert => ({ + orgId: event.orgId, + id: event.id, + actorType: event.actorType, + userId: event.userId ?? null, + apiKeyId: event.apiKeyId ?? null, + actorId: event.actorId ?? null, + actorLabel: event.actorLabel ?? null, + affectedUserId: event.affectedUserId ?? null, + source: event.source, + action: event.action, + outcome: event.outcome, + denialReason: event.denialReason ?? null, + resourceType: event.resourceType ?? null, + resourceId: event.resourceId ?? null, + changedFields: event.changes === undefined ? null : [...event.changes.fields], + changesJson: event.changes ?? null, + metadataJson: event.metadata ?? null, + requestId: event.requestId ?? null, + originIp: event.originIp ?? null, + originCountry: event.originCountry ?? null, + occurredAt: msToDate(event.occurredAtMs), + recordedAt: msToDate(recordedAtMs), +}) diff --git a/apps/api/src/services/audit/audit-log-retention.ts b/apps/api/src/services/audit/audit-log-retention.ts new file mode 100644 index 000000000..ec19b59ff --- /dev/null +++ b/apps/api/src/services/audit/audit-log-retention.ts @@ -0,0 +1,78 @@ +import { auditLogEntries } from "@maple/db" +import { inArray, lt } from "drizzle-orm" +import { Clock, Config, Effect } from "effect" +import { Database } from "@/platform/DatabaseLive" +import { msToDate } from "@/platform/time" + +/** + * Retention for the org audit log (`audit_log_entries`). + * + * Entries older than `AUDIT_LOG_RETENTION_DAYS` (default 400 — a spec-friendly + * 13 months) are swept in bounded batches so one tick never holds its Postgres + * connection for minutes. Runs from the API worker's existing hourly retention + * cron rather than its own schedule — every new cron string costs an entry in + * both `wrangler.jsonc` and `alchemy.run.ts`, and a horizon this wide has no + * reason to tick on a different beat. + */ + +const DEFAULT_RETENTION_DAYS = 400 +const DAY_MS = 24 * 60 * 60 * 1000 + +/** Rows per DELETE, and a per-tick ceiling; the hourly cadence drains any backlog. */ +const RETENTION_BATCH_ROWS = 5_000 +const RETENTION_MAX_BATCHES = 20 + +const retentionDaysConfig = Config.number("AUDIT_LOG_RETENTION_DAYS").pipe( + Config.withDefault(DEFAULT_RETENTION_DAYS), +) + +/** + * Apply retention. Every batch runs inside ONE `execute`: under `DatabasePgLive` + * each call dials and tears down its own postgres.js client, so the handshake + * count is what costs, not the statement count. + */ +export const runAuditLogRetention = Effect.gen(function* () { + const retentionDays = yield* retentionDaysConfig + const now = yield* Clock.currentTimeMillis + const cutoff = msToDate(now - retentionDays * DAY_MS) + const database = yield* Database + + const deleted = yield* database.execute(async (db) => { + let total = 0 + for (let batch = 0; batch < RETENTION_MAX_BATCHES; batch++) { + const staleIds = db + .select({ id: auditLogEntries.id }) + .from(auditLogEntries) + .where(lt(auditLogEntries.occurredAt, cutoff)) + .limit(RETENTION_BATCH_ROWS) + const rows = await db + .delete(auditLogEntries) + .where(inArray(auditLogEntries.id, staleIds)) + .returning({ id: auditLogEntries.id }) + total += rows.length + if (rows.length < RETENTION_BATCH_ROWS) break + } + return total + }) + + yield* Effect.annotateCurrentSpan({ + "audit.retention.deleted": deleted, + "audit.retention.days": retentionDays, + "audit.retention.outcome": "completed", + }) + yield* Effect.logInfo("[audit] log retention tick complete").pipe( + Effect.annotateLogs({ deleted, retentionDays }), + ) +}).pipe( + // tapCause lets the cause propagate so `withSpan` marks the tick as Error. + Effect.tapCause((cause) => + Effect.annotateCurrentSpan({ "audit.retention.outcome": "failed" }).pipe( + Effect.flatMap(() => + Effect.logError("[audit] log retention tick failed").pipe( + Effect.annotateLogs({ error: String(cause) }), + ), + ), + ), + ), + Effect.withSpan("AuditLogRetention.tick"), +) diff --git a/apps/api/src/services/auth/ApiAuthorizationLayer.ts b/apps/api/src/services/auth/ApiAuthorizationLayer.ts index 8db63c1ac..37f45cfde 100644 --- a/apps/api/src/services/auth/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationLayer.ts @@ -4,6 +4,8 @@ import { Effect, Layer, Option, Schema } from "effect" import { ApiKeysService } from "@/services/org/ApiKeysService" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "@/services/auth/auth-span" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogService } from "@/services/audit/AuditLogService" import { Env } from "@/platform/Env" const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) @@ -22,6 +24,7 @@ export const ApiAuthorizationLayer = Layer.effect( Effect.gen(function* () { const env = yield* Env const apiKeys = yield* ApiKeysService + const audit = yield* AuditLogService const resolveTenant = makeResolveTenant(env) return CurrentTenant.Authorization.of({ @@ -42,12 +45,30 @@ export const ApiAuthorizationLayer = Layer.effect( if (Option.isSome(apiKeyResolved)) { const resolved = apiKeyResolved.value + // Denied attempts are audited with the same attribution as + // successes — a key probing a surface it is not valid for is + // exactly what the audit log exists to surface. + const recordDenied = (denialReason: string) => + audit.record({ + orgId: resolved.orgId, + actor: { + type: "api_key", + userId: resolved.userId, + apiKeyId: resolved.keyId, + }, + source: "api", + action: "api.request", + outcome: "denied", + denialReason, + }) if (resolved.kind !== "standard") { + yield* recordDenied("This API key is only valid for the MCP server") return yield* new UnauthorizedError({ message: "This API key is only valid for the MCP server", }) } if (resolved.scopes !== null) { + yield* recordDenied("Restricted API keys must use the /v2 API") return yield* new UnauthorizedError({ message: "Restricted API keys must use the /v2 API", }) @@ -63,15 +84,20 @@ export const ApiAuthorizationLayer = Layer.effect( roles: resolved.roles ?? apiKeyDefaultRoles, authMode: "self_hosted", }) - return yield* Effect.provideService(httpEffect, CurrentTenant.Context, tenant) + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, tenant), + Effect.provideService(CurrentAuditActor, { + type: "api_key", + apiKeyId: resolved.keyId, + }), + ) } const tenant = yield* resolveTenant(request.headers) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema(tenant), + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), + Effect.provideService(CurrentAuditActor, { type: "user" }), ) }), }) diff --git a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts index 10010fa08..83117610d 100644 --- a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts @@ -15,6 +15,8 @@ import { ORG_SELECTION_HEADER } from "@maple/auth" import { makeResolveTenant } from "./AuthService" import { OrgMembershipService } from "@/services/auth/OrgMembershipService" import { annotateAuthSpan } from "@/services/auth/auth-span" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogService } from "@/services/audit/AuditLogService" import { Env } from "@/platform/Env" import { API_V2_RATE_LIMIT_PERIOD_SECONDS, @@ -57,6 +59,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( const env = yield* Env const apiKeys = yield* ApiKeysService const rateLimiter = yield* ApiV2RateLimiter + const audit = yield* AuditLogService // The one resolver wired for organization selection: `x-maple-org-id` is // a v2-client affordance (the iOS app publishing a widget snapshot per // organization), and every other resolver rejects the header instead. @@ -128,13 +131,38 @@ export const ApiAuthorizationV2Layer = Layer.effect( ) } + // A refused attempt is the highest-signal audit row there is — + // denials are recorded with the same actor attribution as + // successes, tagged `outcome: "denied"`. + const recordDenied = (denialReason: string) => + audit.record({ + orgId: resolved.orgId, + actor: { + type: "api_key", + userId: resolved.userId, + apiKeyId: resolved.keyId, + }, + source: "api", + action: "api.request", + outcome: "denied", + denialReason, + metadata: { method: request.method, path: requestPath(request.url) }, + ...(request.headers["cf-ray"] !== undefined + ? { requestId: request.headers["cf-ray"] } + : undefined), + ...(request.headers["cf-connecting-ip"] !== undefined + ? { originIp: request.headers["cf-connecting-ip"] } + : undefined), + ...(request.headers["cf-ipcountry"] !== undefined + ? { originCountry: request.headers["cf-ipcountry"] } + : undefined), + }) + const required = requiredScopeForRequest(request.method, requestPath(request.url)) if (required !== null && !scopeAllows(resolved.scopes, required)) { - return yield* Effect.fail( - V2InsufficientScope.make( - `This API key does not have the "${required.family}:${required.access}" scope required for this request.`, - ), - ) + const message = `This API key does not have the "${required.family}:${required.access}" scope required for this request.` + yield* recordDenied(message) + return yield* Effect.fail(V2InsufficientScope.make(message)) } // An API key is already organization-bound, so a selection could @@ -143,11 +171,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( // check has to be here too. const requestedOrg = getOrgSelectionHeader(request.headers) if (requestedOrg !== undefined && requestedOrg !== resolved.orgId) { - return yield* Effect.fail( - V2OrganizationAccessDenied.make( - "An API key cannot select a different organization.", - ), - ) + const message = "An API key cannot select a different organization." + yield* recordDenied(message) + return yield* Effect.fail(V2OrganizationAccessDenied.make(message)) } const tenant = new CurrentTenant.TenantSchema({ @@ -157,7 +183,13 @@ export const ApiAuthorizationV2Layer = Layer.effect( authMode: "self_hosted", ...(resolved.scopes !== null ? { scopes: resolved.scopes } : undefined), }) - return yield* Effect.provideService(httpEffect, CurrentTenant.Context, tenant) + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, tenant), + Effect.provideService(CurrentAuditActor, { + type: "api_key", + apiKeyId: resolved.keyId, + }), + ) } const tenant = yield* resolveTenant(request.headers).pipe( @@ -166,10 +198,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( ), ) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema(tenant), + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), + Effect.provideService(CurrentAuditActor, { type: "user" }), ) }), }) diff --git a/apps/api/src/services/auth/SessionAuthorizationLayer.ts b/apps/api/src/services/auth/SessionAuthorizationLayer.ts index 3917091d7..82a706283 100644 --- a/apps/api/src/services/auth/SessionAuthorizationLayer.ts +++ b/apps/api/src/services/auth/SessionAuthorizationLayer.ts @@ -4,6 +4,7 @@ import { CurrentTenant } from "@maple/domain/http" import { Effect, Layer } from "effect" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "@/services/auth/auth-span" +import { CurrentAuditActor } from "@/services/auth/audit-actor" import { Env } from "@/platform/Env" const getBearerToken = (headers: Record): string | undefined => { @@ -47,10 +48,9 @@ export const SessionAuthorizationLayer = Layer.effect( const tenant = yield* resolveTenant(request.headers) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) - return yield* Effect.provideService( - httpEffect, - CurrentTenant.Context, - new CurrentTenant.TenantSchema(tenant), + return yield* httpEffect.pipe( + Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), + Effect.provideService(CurrentAuditActor, { type: "user" }), ) }), }) diff --git a/apps/api/src/services/auth/audit-actor.ts b/apps/api/src/services/auth/audit-actor.ts new file mode 100644 index 000000000..5258a336b --- /dev/null +++ b/apps/api/src/services/auth/audit-actor.ts @@ -0,0 +1,23 @@ +import { Context } from "effect" +import type { ApiKeyId } from "@maple/domain/primitives" + +/** + * How the current HTTP request authenticated, for audit attribution. The + * tenant context deliberately does not say whether a request came from a + * dashboard session or an API key — this reference carries that one fact. + */ +export interface AuditActorInfo { + readonly type: "user" | "api_key" + readonly apiKeyId?: ApiKeyId +} + +/** + * A reference (typed default, no handler requirement) rather than a service: + * the auth middlewares override it per request, and handlers that never record + * audit entries are unaffected. `undefined` means the request skipped the + * standard auth middlewares (internal tokens, tests). + */ +export class CurrentAuditActor extends Context.Reference( + "@maple/api/services/auth/CurrentAuditActor", + { defaultValue: () => undefined }, +) {} diff --git a/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts b/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts index 8e6120f40..e2002a232 100644 --- a/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts @@ -7,6 +7,7 @@ import { Clock, Effect, Layer, Schema } from "effect" import { Database } from "@/platform/DatabaseLive" import { msToDate } from "@/platform/time" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import { WarehouseQueryService, type WarehouseQueryServiceApi, @@ -66,7 +67,11 @@ const makeWarehouseStub = (contexts: Array): WarehouseQueryServiceApi => const makeLayer = (contexts: Array) => { const database = createTestDb(createdDbs).layer const actors = ErrorActorsService.layer.pipe(Layer.provide(database)) - const workflow = ErrorIssueWorkflowService.layer.pipe(Layer.provide(database), Layer.provide(actors)) + const workflow = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layer), + Layer.provide(database), + Layer.provide(actors), + ) const warehouse = Layer.succeed(WarehouseQueryService, makeWarehouseStub(contexts)) const readModels = readRequirements.pipe( Layer.provide(database), diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts index a4ca5ae30..b3cbb77dd 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts @@ -18,13 +18,17 @@ import { import { and, eq } from "drizzle-orm" import { Database } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ErrorActorsService } from "./ErrorActorsService" import { ErrorIssueWorkflowService } from "./ErrorIssueWorkflowService" // Compile-time guard: broadening this service to warehouse, cache, Env, // notifications, or WorkerEnvironment makes this assignment fail. -const databaseAndActorsOnly: Layer.Layer = - ErrorIssueWorkflowService.layer +const databaseAndActorsOnly: Layer.Layer< + ErrorIssueWorkflowService, + never, + Database | ErrorActorsService | AuditLogService +> = ErrorIssueWorkflowService.layer const asOrgId = Schema.decodeUnknownSync(OrgId) const asPullRequestId = Schema.decodeUnknownSync(ErrorIssuePullRequestId) @@ -42,7 +46,8 @@ afterEach(() => cleanupTestDbs(createdDbs)) const makeLayer = () => { const database = createTestDb(createdDbs).layer const actors = ErrorActorsService.layer.pipe(Layer.provide(database)) - const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(database, actors))) + const audit = AuditLogService.layer.pipe(Layer.provide(database)) + const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(database, actors, audit))) return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(database)) } diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts index 875d91b8e..2e97446c7 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts @@ -22,7 +22,9 @@ import { CLOSED_WORKFLOW_STATES, MACHINE_OWNED_WORKFLOW_STATES, } from "@maple/domain/http" +import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" import { + actors, alertIncidents, errorIncidents, errorIssues, @@ -37,6 +39,7 @@ import { import { and, desc, eq, inArray, sql } from "drizzle-orm" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import { Database } from "@/platform/DatabaseLive" +import { AuditLogService } from "@/services/audit/AuditLogService" import { readTxid, txidColumn } from "@/platform/electric-txid" import { dateToMs, msToDate } from "@/platform/time" import { ErrorActorsService } from "./ErrorActorsService" @@ -169,10 +172,15 @@ export interface ErrorIssueWorkflowServiceApi extends ErrorIssueWorkflowPublicAp > } -const make: Effect.Effect = Effect.gen( +const make: Effect.Effect< + ErrorIssueWorkflowServiceApi, + never, + Database | ErrorActorsService | AuditLogService +> = Effect.gen( function* () { const database = yield* Database - const actors = yield* ErrorActorsService + const actorsService = yield* ErrorActorsService + const audit = yield* AuditLogService const dbExecute = makeErrorDatabaseExecute(database, "ErrorIssueWorkflowService") const newEventId = () => decodeEventIdSync(randomUUID()) @@ -376,7 +384,7 @@ const make: Effect.Effect row.id) const openSet = yield* issuesWithOpenIncidents(orgId, issueIds) const activityMap = yield* issueActivityRollups(orgId, issueIds) - const actorMap = yield* actors.collectActorDocs( + const actorMap = yield* actorsService.collectActorDocs( orgId, rows.flatMap((row) => [row.assignedActorId ?? null, row.leaseHolderActorId ?? null]), ) @@ -392,6 +400,62 @@ const make: Effect.Effect + Effect.gen(function* () { + const rows = yield* dbExecute((db) => + db + .select() + .from(actors) + .where(and(eq(actors.orgId, orgId), eq(actors.id, actorId))) + .limit(1), + ) + const actor = rows[0] + if (actor === undefined || (actor.type !== "agent" && actor.type !== "user")) return + yield* audit.record({ + orgId, + // A human actor at this layer may have acted from the dashboard or + // over MCP — the issue event does not say which. + actor: + actor.type === "agent" + ? { + type: "agent", + actorId, + ...(actor.agentName === null ? undefined : { label: actor.agentName }), + // On-behalf-of: the human who registered the agent, the + // closest authority the actor registry records. + ...(actor.createdBy === null ? undefined : { userId: actor.createdBy }), + } + : { + type: "user", + ...(actor.userId === null ? undefined : { userId: actor.userId }), + actorId, + }, + source: actor.type === "agent" ? "mcp" : "dashboard", + action: `error_issue.${type}`, + resourceType: "error_issue", + resourceId: encodePublicId(PublicIdPrefixes.errorIssue, issueId), + metadata: { + ...(opts.fromState != null ? { from_state: opts.fromState } : undefined), + ...(opts.toState != null ? { to_state: opts.toState } : undefined), + }, + }) + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Issue event audit write failed", { issueId, cause }), + ), + ) + const recordEvent: ErrorIssueWorkflowServiceApi["recordEvent"] = Effect.fn( "ErrorsService.recordEvent", )(function* (orgId, issueId, actorId, type, opts = {}) { @@ -407,7 +471,12 @@ const make: Effect.Effect db.insert(errorIssueEvents).values(insert)) + const inserted = yield* dbExecute((db) => db.insert(errorIssueEvents).values(insert)) + // System/sweep events carry no actor and stay out of the audit log. + if (actorId !== null) { + yield* recordEventAudit(orgId, issueId, actorId, type, opts) + } + return inserted }) /** @@ -525,7 +594,7 @@ const make: Effect.Effect db.insert(errorIssueEvents).values(row)) - yield* actors.touchActor(orgId, actorId, timestamp) - const actorMap = yield* actors.collectActorDocs(orgId, [actorId]) + yield* actorsService.touchActor(orgId, actorId, timestamp) + const actorMap = yield* actorsService.collectActorDocs(orgId, [actorId]) return rowToEvent(row, actorMap) }) @@ -811,7 +880,7 @@ const make: Effect.Effect row.actorId ?? null), ) diff --git a/apps/api/src/services/errors/ErrorsService.test.ts b/apps/api/src/services/errors/ErrorsService.test.ts index d85917369..bb816e014 100644 --- a/apps/api/src/services/errors/ErrorsService.test.ts +++ b/apps/api/src/services/errors/ErrorsService.test.ts @@ -38,6 +38,7 @@ import { Database, DatabaseError } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { isRetryablePostgresContention } from "@/platform/postgres-errors" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import type { SqlQueryOptions, WarehouseQueryServiceApi } from "@/services/warehouse/WarehouseQueryService" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { ErrorActorsService } from "./ErrorActorsService" @@ -209,6 +210,7 @@ const makeErrorsLayer = ( const databaseLive = testDb.layer const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const errorIssueWorkflowLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layer), Layer.provide(databaseLive), Layer.provide(errorActorsLive), ) @@ -299,6 +301,7 @@ const makeGatingLayer = (opts: { const databaseLive = testDb.layer const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const errorIssueWorkflowLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layer), Layer.provide(databaseLive), Layer.provide(errorActorsLive), ) diff --git a/apps/api/src/services/errors/IssueFixVerificationService.test.ts b/apps/api/src/services/errors/IssueFixVerificationService.test.ts index 6383d30c5..43799cfc8 100644 --- a/apps/api/src/services/errors/IssueFixVerificationService.test.ts +++ b/apps/api/src/services/errors/IssueFixVerificationService.test.ts @@ -8,6 +8,7 @@ import { eq } from "drizzle-orm" import { Database } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { AuditLogService } from "@/services/audit/AuditLogService" import { ErrorActorsService } from "./ErrorActorsService" import { ErrorIssueWorkflowService } from "./ErrorIssueWorkflowService" import { PullRequestLookup } from "./PullRequestLookup" @@ -64,6 +65,7 @@ const makeLayer = (lookup?: { const envLive = Env.layer.pipe(Layer.provide(testConfig())) const actorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const workflowLive = ErrorIssueWorkflowService.layer.pipe( + Layer.provide(AuditLogService.layer), Layer.provide(databaseLive), Layer.provide(actorsLive), ) diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts index 1094eddf1..fbe65b871 100644 --- a/apps/api/src/vcs-sync-runtime.ts +++ b/apps/api/src/vcs-sync-runtime.ts @@ -4,6 +4,7 @@ import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" import { Cause, Effect, Layer, Option } from "effect" import { layerPg } from "@/platform/DatabasePgLive" +import { AuditLogService } from "@/services/audit/AuditLogService" import { Env } from "@/platform/Env" import { GithubAppClient } from "./services/integrations/vcs/vendor/github/GithubAppClient" import { GithubHttp } from "./services/integrations/vcs/vendor/github/GithubHttp" @@ -56,7 +57,9 @@ export const buildVcsSyncLayer = (_env: Record) => { // rather than in `Base`, keeping the cron layer as light as it was. const ErrorActorsServiceLive = ErrorActorsService.layer.pipe(Layer.provide(Base)) const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(Layer.mergeAll(Base, ErrorActorsServiceLive)), + Layer.provide( + Layer.mergeAll(Base, ErrorActorsServiceLive, AuditLogService.layer.pipe(Layer.provide(Base))), + ), ) const IssueFixVerificationServiceLive = IssueFixVerificationService.layer.pipe( Layer.provide( diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index b296a4bb3..523c11ce0 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -439,6 +439,17 @@ const handleQueue = async ( } return } + if (queueKind === "audit-events") { + const { buildAuditEventsLayer, processAuditEventsBatch, flushAuditEventsTelemetry } = await import( + "./audit-events-runtime" + ) + try { + await runScheduledEffect(buildAuditEventsLayer(env), await scoped(processAuditEventsBatch(batch)), ctx) + } finally { + ctx.waitUntil(flushAuditEventsTelemetry(env)) + } + return + } if (queueKind === "unknown") { throw new Error(`No queue consumer configured for "${batch.queue}"`) } @@ -475,14 +486,20 @@ const handleScheduled = async ( const { runScrapeCheckRetention } = await import("@/services/integrations/scrape-check-retention") const { runPlanetScaleEventRetention } = await import("@/services/integrations/planetscale-event-retention") + const { runAuditLogRetention } = await import("@/services/audit/audit-log-retention") try { - // Both sweeps ride this one cron: each new cron string costs an entry in - // wrangler.jsonc and alchemy.run.ts, and neither needs its own beat. + // All three sweeps ride this one cron: each new cron string costs an entry + // in wrangler.jsonc and alchemy.run.ts, and none needs its own beat. // Sequential, not concurrent — they share one Postgres socket for the // whole tick, so running them concurrently would only queue on it. await runScheduledEffect( buildScrapeRetentionLayer(env), - await scoped(Effect.andThen(runScrapeCheckRetention, runPlanetScaleEventRetention)), + await scoped( + Effect.andThen( + runScrapeCheckRetention, + Effect.andThen(runPlanetScaleEventRetention, runAuditLogRetention), + ), + ), ctx, { onInterrupt: "graceful" }, ) diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index cb25adbaf..998325d98 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -16,6 +16,7 @@ "API_V2_RATE_LIMIT_PARTITION": "local", "PLANETSCALE_WEBHOOK_QUEUE_NAME": "maple-planetscale-webhooks-local", "VCS_SYNC_QUEUE_NAME": "maple-vcs-sync-local", + "AUDIT_EVENTS_QUEUE_NAME": "maple-audit-events-local", }, "ratelimits": [ { @@ -99,6 +100,7 @@ "binding": "PLANETSCALE_WEBHOOK_QUEUE", "queue": "maple-planetscale-webhooks-local", }, + { "binding": "AUDIT_EVENTS_QUEUE", "queue": "maple-audit-events-local" }, ], "consumers": [ { @@ -113,6 +115,12 @@ "max_batch_timeout": 5, "max_retries": 3, }, + { + "queue": "maple-audit-events-local", + "max_batch_size": 25, + "max_batch_timeout": 5, + "max_retries": 5, + }, ], }, } diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx new file mode 100644 index 000000000..967e28e92 --- /dev/null +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -0,0 +1,337 @@ +import type { AuditActorType, AuditOutcome } from "@maple/domain/http" +import type { V2AuditChanges, V2AuditLogEntry } from "@maple/domain/http/v2" +import { useState, type ReactNode } from "react" + +import { Result, useAtomRefresh, useAtomValue } from "@/lib/effect-atom" +import { auditLogPageAtom } from "@/lib/services/atoms/audit-log-atoms" + +import { Badge } from "@maple/ui/components/ui/badge" +import { Button } from "@maple/ui/components/ui/button" +import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { cn } from "@maple/ui/lib/utils" +import { formatRelativeTime } from "@maple/ui/lib/time-format" +import { AlertWarningIcon, HistoryIcon } from "@/components/icons" + +type ActorFilter = AuditActorType | "all" +type OutcomeFilter = AuditOutcome | "all" + +const ACTOR_FILTERS: ReadonlyArray<{ value: ActorFilter; label: string }> = [ + { value: "all", label: "All" }, + { value: "user", label: "Users" }, + { value: "api_key", label: "API keys" }, + { value: "agent", label: "Agents" }, + { value: "system", label: "System" }, +] + +const OUTCOME_FILTERS: ReadonlyArray<{ value: OutcomeFilter; label: string }> = [ + { value: "all", label: "All" }, + { value: "allowed", label: "Allowed" }, + { value: "denied", label: "Denied" }, +] + +const ACTOR_BADGES: Record = { + user: { label: "User", variant: "secondary" }, + api_key: { label: "API key", variant: "success" }, + agent: { label: "Agent", variant: "info" }, + system: { label: "System", variant: "outline" }, +} satisfies Record + +// Shared column lanes so the header row and entry rows stay aligned. Resource and +// source collapse on narrower viewports; time + actor + action always stay visible. +const COL = { + time: "w-[96px] shrink-0", + actor: "w-[200px] min-w-0 shrink-0", + action: "min-w-0 flex-1", + resource: "hidden w-[220px] min-w-0 shrink-0 md:block", + source: "hidden w-[80px] shrink-0 lg:block", +} +const COL_HEADER = "text-muted-foreground/70 font-mono text-[10px] uppercase tracking-[0.12em]" + +function formatDateTime(value: string): string { + return new Date(value).toLocaleString(undefined, { + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }) +} + +// JSON.stringify(undefined) is undefined — surface it as text in tooltips. +function formatChangeValue(value: unknown): string { + return JSON.stringify(value) ?? "undefined" +} + +function formatChangesTooltip(changes: V2AuditChanges): string { + return changes.fields + .map( + (field) => + `${field}: ${formatChangeValue(changes.before[field])} → ${formatChangeValue(changes.after[field])}`, + ) + .join("\n") +} + +function formatSourceTooltip(entry: V2AuditLogEntry): string | undefined { + const lines = [ + entry.origin_ip !== null || entry.origin_country !== null + ? `From ${entry.origin_ip ?? "unknown IP"}${entry.origin_country !== null ? ` (${entry.origin_country})` : ""}` + : null, + entry.request_id !== null ? `Request ${entry.request_id}` : null, + ].filter((line) => line !== null) + return lines.length > 0 ? lines.join("\n") : undefined +} + +interface AuditLogView { + source: { data: ReadonlyArray } + entries: V2AuditLogEntry[] + hasMore: boolean + nextCursor: string | null +} + +export function AuditLogSection() { + const [actorFilter, setActorFilter] = useState("all") + const [outcomeFilter, setOutcomeFilter] = useState("all") + const [cursor, setCursor] = useState(undefined) + + const pageAtom = auditLogPageAtom({ + ...(cursor !== undefined ? { cursor } : undefined), + ...(actorFilter !== "all" ? { actorType: actorFilter } : undefined), + ...(outcomeFilter !== "all" ? { outcome: outcomeFilter } : undefined), + }) + const pageResult = useAtomValue(pageAtom) + const refreshPage = useAtomRefresh(pageAtom) + + // Each Load more / filter change swaps to a new page atom, which starts in its + // initial state. Keep the accumulated entries so the table stays rendered + // (dimmed) while the next page loads; a fresh (cursor-less) page replaces them. + const [view, setView] = useState(null) + if (Result.isSuccess(pageResult) && view?.source !== pageResult.value) { + setView({ + source: pageResult.value, + entries: + cursor === undefined + ? [...pageResult.value.data] + : [...(view?.entries ?? []), ...pageResult.value.data], + hasMore: pageResult.value.has_more, + nextCursor: pageResult.value.next_cursor, + }) + } + + function handleFilterSelect(value: ActorFilter) { + if (value === actorFilter) return + setActorFilter(value) + setCursor(undefined) + } + + function handleOutcomeSelect(value: OutcomeFilter) { + if (value === outcomeFilter) return + setOutcomeFilter(value) + setCursor(undefined) + } + + const waiting = !Result.isSuccess(pageResult) || pageResult.waiting + + return ( +
+
+
+ {ACTOR_FILTERS.map((filter) => ( + handleFilterSelect(filter.value)} + > + {filter.label} + + ))} +
+
+ {OUTCOME_FILTERS.map((filter) => ( + handleOutcomeSelect(filter.value)} + > + {filter.label} + + ))} +
+
+

+ Every change made through the dashboard, API, and MCP. +

+
+ +
+ {view === null && Result.isFailure(pageResult) ? ( + + + + + + Couldn't load the audit log + + Something went wrong while loading audit log entries. + + + + + ) : view === null ? ( +
+ + + +
+ ) : view.entries.length === 0 ? ( + + + + + + No audit log entries + + Actions performed by users, API keys, and agents will appear here. + + + + ) : ( +
+
+ Time + Actor + Action + Resource + Source +
+ {view.entries.map((entry) => ( + + ))} +
+ )} +
+ + {view !== null && view.hasMore && view.nextCursor !== null && ( +
+ Showing {view.entries.length} entries — more available + +
+ )} +
+ ) +} + +function FilterTab({ + active, + onClick, + children, +}: { + active: boolean + onClick: () => void + children: ReactNode +}) { + return ( + + ) +} + +function AuditLogRow({ entry }: { entry: V2AuditLogEntry }) { + const badge = ACTOR_BADGES[entry.actor_type] + const actorLabel = entry.actor_name ?? entry.actor_id ?? "—" + + return ( +
+ + {formatRelativeTime(entry.occurred_at)} + +
+ + {badge.label} + + + {actorLabel} + +
+
+
+ + {entry.action} + + {entry.outcome === "denied" && ( + + Denied + + )} +
+ {entry.outcome === "denied" && entry.denial_reason !== null && ( +

+ {entry.denial_reason} +

+ )} + {entry.changes !== null && entry.changes.fields.length > 0 && ( +

+ {entry.changes.fields.join(", ")} +

+ )} +
+
+ {entry.resource_type !== null || entry.resource_id !== null ? ( +
+ {entry.resource_type !== null && ( + + {entry.resource_type} + + )} + {entry.resource_id !== null && ( + + {entry.resource_id} + + )} +
+ ) : ( + — + )} +
+ + {entry.source} + {entry.origin_country !== null && ( + · {entry.origin_country} + )} + +
+ ) +} diff --git a/apps/web/src/components/settings/settings-nav.tsx b/apps/web/src/components/settings/settings-nav.tsx index 1efc11c9b..c6c8446c7 100644 --- a/apps/web/src/components/settings/settings-nav.tsx +++ b/apps/web/src/components/settings/settings-nav.tsx @@ -14,6 +14,7 @@ import { DatabaseIcon, GearIcon, GridIcon, + HistoryIcon, KeyIcon, ServerIcon, ShieldIcon, @@ -26,6 +27,7 @@ import { SettingsNavShell } from "@/components/settings/settings-nav-shell" export const settingsTabValues = [ "organization", "members", + "audit-log", "setup-audit", "ingestion", "api-keys", @@ -41,6 +43,7 @@ export type SettingsTab = (typeof settingsTabValues)[number] export const settingsTabLabels: Record = { organization: "Organization", members: "Members", + "audit-log": "Audit Log", "setup-audit": "Setup Audit", ingestion: "Ingestion", "api-keys": "API Keys", @@ -108,6 +111,7 @@ const navSections: SettingsNavSection[] = [ items: [ { id: "organization", label: "Organization", icon: GearIcon }, { id: "members", label: "Members", icon: UserIcon }, + { id: "audit-log", label: "Audit Log", icon: HistoryIcon }, // Spans alerting, ingestion and integrations, so it sits at workspace level rather than // under any one of them. { id: "setup-audit", label: "Setup Audit", icon: CircleCheckIcon }, diff --git a/apps/web/src/lib/services/atoms/audit-log-atoms.ts b/apps/web/src/lib/services/atoms/audit-log-atoms.ts new file mode 100644 index 000000000..72772d2cf --- /dev/null +++ b/apps/web/src/lib/services/atoms/audit-log-atoms.ts @@ -0,0 +1,46 @@ +import type { AuditActorType, AuditOutcome } from "@maple/domain/http" +import { Effect } from "effect" +import { Atom } from "@/lib/effect-atom" +import { MapleApiV2AtomClient } from "@/lib/services/common/v2-atom-client" + +export const AUDIT_LOG_PAGE_LIMIT = 50 + +const ACTOR_TYPES: ReadonlyArray = ["user", "api_key", "agent", "system"] +const OUTCOMES: ReadonlyArray = ["allowed", "denied"] + +export interface AuditLogPageInput { + readonly cursor?: string + readonly actorType?: AuditActorType + readonly outcome?: AuditOutcome +} + +// Actor types and outcomes never contain "|", and the cursor is the trailing +// segment, so splitting on the first two separators stays unambiguous even for +// exotic cursors. +const family = Atom.family((key: string) => { + const firstSeparator = key.indexOf("|") + const secondSeparator = key.indexOf("|", firstSeparator + 1) + const actorRaw = key.slice(0, firstSeparator) + const outcomeRaw = key.slice(firstSeparator + 1, secondSeparator) + const cursor = key.slice(secondSeparator + 1) + const actorType = ACTOR_TYPES.find((type) => type === actorRaw) + const outcome = OUTCOMES.find((value) => value === outcomeRaw) + + return MapleApiV2AtomClient.runtime.atom( + Effect.gen(function* () { + const client = yield* MapleApiV2AtomClient + return yield* client.auditLog.list({ + query: { + limit: AUDIT_LOG_PAGE_LIMIT, + ...(cursor !== "" ? { cursor } : undefined), + ...(actorType !== undefined ? { actor_type: actorType } : undefined), + ...(outcome !== undefined ? { outcome } : undefined), + }, + }) + }), + ) +}) + +/** One page of the org's audit log, keyed by cursor + actor-type/outcome filters. */ +export const auditLogPageAtom = (input: AuditLogPageInput) => + family(`${input.actorType ?? ""}|${input.outcome ?? ""}|${input.cursor ?? ""}`) diff --git a/apps/web/src/routes/settings.tsx b/apps/web/src/routes/settings.tsx index 91ecda7a9..e2442ade8 100644 --- a/apps/web/src/routes/settings.tsx +++ b/apps/web/src/routes/settings.tsx @@ -8,6 +8,7 @@ import { BillingSection } from "@/components/settings/billing-section" import { MembersSection } from "@/components/settings/members-section" import { IngestionSection } from "@/components/settings/ingestion-section" import { ApiKeysSection } from "@/components/settings/api-keys-section" +import { AuditLogSection } from "@/components/settings/audit-log-section" import { DeveloperSection } from "@/components/settings/developer-section" import { McpSection } from "@/components/settings/mcp-section" import { NotificationsSection } from "@/components/settings/notifications-section" @@ -135,6 +136,7 @@ function SettingsPage() { {activeTab === "organization" && } {activeTab === "members" && } + {activeTab === "audit-log" && } {activeTab === "setup-audit" && } {activeTab === "ingestion" && } {activeTab === "api-keys" && } diff --git a/packages/db/drizzle/0050_audit_log_entries.sql b/packages/db/drizzle/0050_audit_log_entries.sql new file mode 100644 index 000000000..fedc989f5 --- /dev/null +++ b/packages/db/drizzle/0050_audit_log_entries.sql @@ -0,0 +1,31 @@ +CREATE TABLE "audit_log_entries" ( + "org_id" text NOT NULL, + "id" text NOT NULL, + "actor_type" text NOT NULL, + "user_id" text, + "api_key_id" text, + "actor_id" text, + "actor_label" text, + "affected_user_id" text, + "source" text NOT NULL, + "action" text NOT NULL, + "outcome" text NOT NULL, + "denial_reason" text, + "resource_type" text, + "resource_id" text, + "changed_fields" text[], + "changes_json" jsonb, + "metadata_json" jsonb, + "request_id" text, + "origin_ip" text, + "origin_country" text, + "occurred_at" timestamp with time zone NOT NULL, + "recorded_at" timestamp with time zone NOT NULL, + CONSTRAINT "audit_log_entries_org_id_id_pk" PRIMARY KEY("org_id","id") +); +--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_occurred_idx" ON "audit_log_entries" USING btree ("org_id","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_actor_type_occurred_idx" ON "audit_log_entries" USING btree ("org_id","actor_type","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_resource_idx" ON "audit_log_entries" USING btree ("org_id","resource_type","resource_id");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_request_idx" ON "audit_log_entries" USING btree ("org_id","request_id");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_outcome_occurred_idx" ON "audit_log_entries" USING btree ("org_id","outcome","occurred_at"); \ No newline at end of file diff --git a/packages/db/drizzle/meta/0050_snapshot.json b/packages/db/drizzle/meta/0050_snapshot.json new file mode 100644 index 000000000..46076f621 --- /dev/null +++ b/packages/db/drizzle/meta/0050_snapshot.json @@ -0,0 +1,8999 @@ +{ + "id": "2b89a081-0fdd-4565-9366-89077aa29ec5", + "prevId": "d60d7088-c27b-48cd-94b6-6d9fd59a02ff", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_triage_settings": { + "name": "ai_triage_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "max_runs_per_day": { + "name": "max_runs_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20 + }, + "max_passes_per_day": { + "name": "max_passes_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 90 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_delivery_events": { + "name": "alert_delivery_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_message": { + "name": "provider_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_reference": { + "name": "provider_reference", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_delivery_events_org_idx": { + "name": "alert_delivery_events_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_org_incident_idx": { + "name": "alert_delivery_events_org_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_due_idx": { + "name": "alert_delivery_events_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_claim_idx": { + "name": "alert_delivery_events_claim_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_delivery_attempt_idx": { + "name": "alert_delivery_events_delivery_attempt_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_destinations": { + "name": "alert_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "config_json": { + "name": "config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_tested_at": { + "name": "last_tested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_error": { + "name": "last_test_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_destinations_org_idx": { + "name": "alert_destinations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_enabled_idx": { + "name": "alert_destinations_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_name_idx": { + "name": "alert_destinations_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_incidents": { + "name": "alert_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_key": { + "name": "incident_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_name": { + "name": "rule_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_delivered_event_type": { + "name": "last_delivered_event_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_incidents_org_idx": { + "name": "alert_incidents_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_status_idx": { + "name": "alert_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_rule_idx": { + "name": "alert_incidents_org_rule_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "rule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_issue_idx": { + "name": "alert_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_incident_key_idx": { + "name": "alert_incidents_incident_key_idx", + "columns": [ + { + "expression": "incident_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_claims": { + "name": "alert_rule_claims", + "schema": "", + "columns": { + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_claims_org_idx": { + "name": "alert_rule_claims_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_states": { + "name": "alert_rule_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'__total__'" + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_states_org_idx": { + "name": "alert_rule_states_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "alert_rule_states_org_id_rule_id_group_key_pk": { + "name": "alert_rule_states_org_id_rule_id_group_key_pk", + "columns": [ + "org_id", + "rule_id", + "group_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rules": { + "name": "alert_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_template_json": { + "name": "notification_template_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_names_json": { + "name": "service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "exclude_service_names_json": { + "name": "exclude_service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "environments_json": { + "name": "environments_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tags_json": { + "name": "tags_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "window_minutes": { + "name": "window_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minimum_sample_count": { + "name": "minimum_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_breaches_required": { + "name": "consecutive_breaches_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "consecutive_healthy_required": { + "name": "consecutive_healthy_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "renotify_interval_minutes": { + "name": "renotify_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 30 + }, + "apdex_threshold_ms": { + "name": "apdex_threshold_ms", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "query_builder_draft_json": { + "name": "query_builder_draft_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "raw_query_sql": { + "name": "raw_query_sql", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "group_by": { + "name": "group_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "query_spec_json": { + "name": "query_spec_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "reducer": { + "name": "reducer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sample_count_strategy": { + "name": "sample_count_strategy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "no_data_behavior": { + "name": "no_data_behavior", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rules_org_idx": { + "name": "alert_rules_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_enabled_idx": { + "name": "alert_rules_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_name_idx": { + "name": "alert_rules_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_settings": { + "name": "anomaly_detector_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sensitivity": { + "name": "sensitivity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'normal'" + }, + "muted_signals_json": { + "name": "muted_signals_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_tick_at": { + "name": "last_tick_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_states": { + "name": "anomaly_detector_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_incident_id": { + "name": "last_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_detector_states_open_incident_idx": { + "name": "anomaly_detector_states_open_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "open_incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"anomaly_detector_states\".\"open_incident_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_detector_states_evaluated_idx": { + "name": "anomaly_detector_states_evaluated_idx", + "columns": [ + { + "expression": "last_evaluated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "anomaly_detector_states_org_id_detector_key_pk": { + "name": "anomaly_detector_states_org_id_detector_key_pk", + "columns": [ + "org_id", + "detector_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_incidents": { + "name": "anomaly_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "opened_value": { + "name": "opened_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_sigma": { + "name": "baseline_sigma", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_value": { + "name": "threshold_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolve_reason": { + "name": "resolve_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "triage_status": { + "name": "triage_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprints_json": { + "name": "fingerprints_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "reopen_count": { + "name": "reopen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_reopened_at": { + "name": "last_reopened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_incidents_org_status_triggered_idx": { + "name": "anomaly_incidents_org_status_triggered_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_triggered_idx": { + "name": "anomaly_incidents_org_triggered_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_detector_idx": { + "name": "anomaly_incidents_org_detector_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "detector_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_issue_idx": { + "name": "anomaly_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_prefix": { + "name": "key_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'standard'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by_email": { + "name": "created_by_email", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_keys_org_id_idx": { + "name": "api_keys_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_log_entries": { + "name": "audit_log_entries", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_label": { + "name": "actor_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "affected_user_id": { + "name": "affected_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "denial_reason": { + "name": "denial_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "changed_fields": { + "name": "changed_fields", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "changes_json": { + "name": "changes_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_ip": { + "name": "origin_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_country": { + "name": "origin_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recorded_at": { + "name": "recorded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "audit_log_entries_org_occurred_idx": { + "name": "audit_log_entries_org_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_actor_type_occurred_idx": { + "name": "audit_log_entries_org_actor_type_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_resource_idx": { + "name": "audit_log_entries_org_resource_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_request_idx": { + "name": "audit_log_entries_org_request_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_outcome_occurred_idx": { + "name": "audit_log_entries_org_outcome_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "outcome", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "audit_log_entries_org_id_id_pk": { + "name": "audit_log_entries_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_analytics_state": { + "name": "cloudflare_analytics_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_at": { + "name": "backfill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "settings_fetched_at": { + "name": "settings_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "quantiles_available": { + "name": "quantiles_available", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovered_at": { + "name": "discovered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "live_scripts_json": { + "name": "live_scripts_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cf_analytics_state_org_dataset_zone_idx": { + "name": "cf_analytics_state_org_dataset_zone_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "zone_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cf_analytics_state_org_idx": { + "name": "cf_analytics_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_hyperdrive_configs": { + "name": "cloudflare_hyperdrive_configs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_host": { + "name": "origin_host", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_port": { + "name": "origin_port", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "origin_scheme": { + "name": "origin_scheme", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_database": { + "name": "origin_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_user": { + "name": "origin_user", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_hyperdrive_configs_org_config_idx": { + "name": "cloudflare_hyperdrive_configs_org_config_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_hyperdrive_configs_org_idx": { + "name": "cloudflare_hyperdrive_configs_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_logpush_connectors": { + "name": "cloudflare_logpush_connectors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'http_requests'" + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_received_at": { + "name": "last_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_rotated_at": { + "name": "secret_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_logpush_connectors_org_idx": { + "name": "cloudflare_logpush_connectors_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_org_enabled_idx": { + "name": "cloudflare_logpush_connectors_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_secret_hash_unique": { + "name": "cloudflare_logpush_connectors_secret_hash_unique", + "columns": [ + { + "expression": "secret_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cli_device_authorizations": { + "name": "cli_device_authorizations", + "schema": "", + "columns": { + "device_code_hash": { + "name": "device_code_hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_code_hash": { + "name": "user_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_name": { + "name": "device_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_roles": { + "name": "approved_roles", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "approved_user_email": { + "name": "approved_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "denied_at": { + "name": "denied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cli_device_authorizations_user_code_unique": { + "name": "cli_device_authorizations_user_code_unique", + "columns": [ + { + "expression": "user_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_device_authorizations_expires_idx": { + "name": "cli_device_authorizations_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_authorizations": { + "name": "mcp_oauth_authorizations", + "schema": "", + "columns": { + "request_id_hash": { + "name": "request_id_hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_code_hash": { + "name": "authorization_code_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_roles": { + "name": "approved_roles", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "approved_user_email": { + "name": "approved_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "denied_at": { + "name": "denied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "used_at": { + "name": "used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_oauth_authorizations_code_unique": { + "name": "mcp_oauth_authorizations_code_unique", + "columns": [ + { + "expression": "authorization_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_authorizations_expires_idx": { + "name": "mcp_oauth_authorizations_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_clients": { + "name": "mcp_oauth_clients", + "schema": "", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "client_uri": { + "name": "client_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_refresh_tokens": { + "name": "mcp_oauth_refresh_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "roles": { + "name": "roles", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_key_id": { + "name": "access_key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replaced_by_id": { + "name": "replaced_by_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_oauth_refresh_tokens_hash_unique": { + "name": "mcp_oauth_refresh_tokens_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_refresh_tokens_family_idx": { + "name": "mcp_oauth_refresh_tokens_family_idx", + "columns": [ + { + "expression": "family_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_refresh_tokens_expires_idx": { + "name": "mcp_oauth_refresh_tokens_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_devices": { + "name": "mobile_devices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bundle_id": { + "name": "bundle_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_version": { + "name": "app_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "live_activity_start_token": { + "name": "live_activity_start_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_name": { + "name": "device_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preferences": { + "name": "preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "disabled_at": { + "name": "disabled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_pushed_at": { + "name": "last_pushed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mobile_devices_org_platform_token_unique": { + "name": "mobile_devices_org_platform_token_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "platform", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mobile_devices_org_idx": { + "name": "mobile_devices_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mobile_devices_user_idx": { + "name": "mobile_devices_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_shares": { + "name": "dashboard_shares", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "widget_id": { + "name": "widget_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_suffix": { + "name": "token_suffix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "dashboard_shares_token_hash_unq": { + "name": "dashboard_shares_token_hash_unq", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_live_unq": { + "name": "dashboard_shares_live_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(widget_id, '')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "revoked_at is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_org_dashboard_idx": { + "name": "dashboard_shares_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_id_idx": { + "name": "dashboard_shares_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dashboard_shares_dashboard_fk": { + "name": "dashboard_shares_dashboard_fk", + "tableFrom": "dashboard_shares", + "tableTo": "dashboards", + "columnsFrom": [ + "org_id", + "dashboard_id" + ], + "columnsTo": [ + "org_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "dashboard_shares_org_id_id_pk": { + "name": "dashboard_shares_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_versions": { + "name": "dashboard_versions", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version_number": { + "name": "version_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "change_kind": { + "name": "change_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_version_id": { + "name": "source_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "dashboard_versions_org_dashboard_idx": { + "name": "dashboard_versions_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_versions_org_dashboard_version_unq": { + "name": "dashboard_versions_org_dashboard_version_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboard_versions_org_id_id_pk": { + "name": "dashboard_versions_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboards": { + "name": "dashboards", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "dashboards_org_updated_idx": { + "name": "dashboards_org_updated_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboards_org_name_idx": { + "name": "dashboards_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboards_org_id_id_pk": { + "name": "dashboards_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.digest_subscriptions": { + "name": "digest_subscriptions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "day_of_week": { + "name": "day_of_week", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "last_sent_at": { + "name": "last_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempted_at": { + "name": "last_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "digest_subscriptions_org_user_idx": { + "name": "digest_subscriptions_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "digest_subscriptions_org_enabled_idx": { + "name": "digest_subscriptions_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.actors": { + "name": "actors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capabilities_json": { + "name": "capabilities_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "actors_org_user_idx": { + "name": "actors_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_agent_name_idx": { + "name": "actors_org_agent_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agent_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_type_idx": { + "name": "actors_org_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_fingerprint_candidates": { + "name": "error_fingerprint_candidates", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_versions_json": { + "name": "service_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_fingerprint_candidates_last_seen_idx": { + "name": "error_fingerprint_candidates_last_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_fingerprint_candidates_org_id_fingerprint_hash_pk": { + "name": "error_fingerprint_candidates_org_id_fingerprint_hash_pk", + "columns": [ + "org_id", + "fingerprint_hash" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_incidents": { + "name": "error_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_incidents_org_issue_idx": { + "name": "error_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_incidents_org_status_idx": { + "name": "error_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_events": { + "name": "error_issue_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_state": { + "name": "from_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "to_state": { + "name": "to_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_events_issue_idx": { + "name": "error_issue_events_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_actor_idx": { + "name": "error_issue_events_actor_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_type_idx": { + "name": "error_issue_events_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_pull_requests": { + "name": "error_issue_pull_requests", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repo_full_name": { + "name": "repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "merge_commit_sha": { + "name": "merge_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "link_source": { + "name": "link_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "linked_by_actor_id": { + "name": "linked_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_pull_requests_issue_pr_idx": { + "name": "error_issue_pull_requests_issue_pr_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo_full_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_pull_requests_repo_number_idx": { + "name": "error_issue_pull_requests_repo_number_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo_full_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_pull_requests_issue_idx": { + "name": "error_issue_pull_requests_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_states": { + "name": "error_issue_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_observed_occurrence_at": { + "name": "last_observed_occurrence_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_issue_states_org_id_issue_id_pk": { + "name": "error_issue_states_org_id_issue_id_pk", + "columns": [ + "org_id", + "issue_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_verifications": { + "name": "error_issue_verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pull_request_id": { + "name": "pull_request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'waiting'" + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "verify_after": { + "name": "verify_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "baseline_versions_json": { + "name": "baseline_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "baseline_occurrence_count": { + "name": "baseline_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "baseline_rate_per_hour": { + "name": "baseline_rate_per_hour", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "verdict": { + "name": "verdict", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "verdict_note": { + "name": "verdict_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "post_merge_occurrence_count": { + "name": "post_merge_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_verifications_due_idx": { + "name": "error_issue_verifications_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "verify_after", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_verifications_issue_idx": { + "name": "error_issue_verifications_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_verifications_open_idx": { + "name": "error_issue_verifications_open_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"error_issue_verifications\".\"status\" in ('waiting', 'running')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issues": { + "name": "error_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'error'" + }, + "source_ref_json": { + "name": "source_ref_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint_version": { + "name": "fingerprint_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_state": { + "name": "workflow_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'triage'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "severity_source": { + "name": "severity_source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_actor_id": { + "name": "assigned_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_holder_actor_id": { + "name": "lease_holder_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolved_by_actor_id": { + "name": "resolved_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_regressed_at": { + "name": "last_regressed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "regression_count": { + "name": "regression_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "seen_versions_json": { + "name": "seen_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "resolved_versions_json": { + "name": "resolved_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "snooze_until": { + "name": "snooze_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issues_org_fp_idx": { + "name": "error_issues_org_fp_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_workflow_idx": { + "name": "error_issues_org_workflow_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_severity_idx": { + "name": "error_issues_org_severity_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "severity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_live_seen_idx": { + "name": "error_issues_org_live_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"error_issues\".\"archived_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_fp_version_idx": { + "name": "error_issues_org_fp_version_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_assignee_idx": { + "name": "error_issues_org_assignee_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "assigned_actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_lease_expiry_idx": { + "name": "error_issues_lease_expiry_idx", + "columns": [ + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_archived_idx": { + "name": "error_issues_org_archived_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"error_issues\".\"archived_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_deliveries": { + "name": "error_notification_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_notification_deliveries_due_idx": { + "name": "error_notification_deliveries_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_notification_deliveries_org_idx": { + "name": "error_notification_deliveries_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_notification_deliveries_key_destination_idx": { + "name": "error_notification_deliveries_key_destination_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_policies": { + "name": "error_notification_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "notify_on_first_seen": { + "name": "notify_on_first_seen", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_regression": { + "name": "notify_on_regression", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_resolve": { + "name": "notify_on_resolve", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_in_review": { + "name": "notify_on_transition_in_review", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_done": { + "name": "notify_on_transition_done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_claim": { + "name": "notify_on_claim", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "min_occurrence_count": { + "name": "min_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'warning'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_tick_states": { + "name": "error_tick_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "processed_through": { + "name": "processed_through", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "bootstrap_completed": { + "name": "bootstrap_completed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "claim_token": { + "name": "claim_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_tick_states_claim_idx": { + "name": "error_tick_states_claim_idx", + "columns": [ + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalation_policies": { + "name": "issue_escalation_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "rules_json": { + "name": "rules_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalations": { + "name": "issue_escalations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "delivery_results_json": { + "name": "delivery_results_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "issue_escalations_dedupe_idx": { + "name": "issue_escalations_dedupe_idx", + "columns": [ + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_due_idx": { + "name": "issue_escalations_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_org_issue_idx": { + "name": "issue_escalations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigation_lens_runs": { + "name": "investigation_lens_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "lens_id": { + "name": "lens_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "verdict": { + "name": "verdict", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "claim": { + "name": "claim", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress_note": { + "name": "progress_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "elapsed_ms": { + "name": "elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "lens_name": { + "name": "lens_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lens_question": { + "name": "lens_question", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deadline_hit": { + "name": "deadline_hit", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "hypothesis_json": { + "name": "hypothesis_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_json": { + "name": "evidence_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mechanism": { + "name": "mechanism", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "self_doubt": { + "name": "self_doubt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suggested_actions_json": { + "name": "suggested_actions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reported_at": { + "name": "reported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ranked_at": { + "name": "ranked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigation_lens_runs_lens_idx": { + "name": "investigation_lens_runs_lens_idx", + "columns": [ + { + "expression": "investigation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lens_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigation_lens_runs_org_inv_idx": { + "name": "investigation_lens_runs_org_inv_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "investigation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "investigation_lens_runs_investigation_id_investigations_id_fk": { + "name": "investigation_lens_runs_investigation_id_investigations_id_fk", + "tableFrom": "investigation_lens_runs", + "tableTo": "investigations", + "columnsFrom": [ + "investigation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigations": { + "name": "investigations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'investigating'" + }, + "seeded_by": { + "name": "seeded_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user'" + }, + "subject_json": { + "name": "subject_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "incident_kind": { + "name": "incident_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "report_json": { + "name": "report_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fanout_state": { + "name": "fanout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "fanout_size": { + "name": "fanout_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "plan_json": { + "name": "plan_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "planner_model": { + "name": "planner_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "planner_elapsed_ms": { + "name": "planner_elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "validator_note": { + "name": "validator_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validator_elapsed_ms": { + "name": "validator_elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "fanout_deadline_at": { + "name": "fanout_deadline_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fanout_attempt": { + "name": "fanout_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "autonomous_turns": { + "name": "autonomous_turns", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "diagnosed_at": { + "name": "diagnosed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigations_incident_idx": { + "name": "investigations_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"investigations\".\"incident_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_created_idx": { + "name": "investigations_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_issue_idx": { + "name": "investigations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_status_idx": { + "name": "investigations_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.live_activities": { + "name": "live_activities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_id": { + "name": "device_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activity_id": { + "name": "activity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "push_token": { + "name": "push_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ended_reason": { + "name": "ended_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "live_activities_device_incident_unique": { + "name": "live_activities_device_incident_unique", + "columns": [ + { + "expression": "device_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "live_activities_incident_idx": { + "name": "live_activities_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_auth_states": { + "name": "oauth_auth_states", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "initiated_by_user_id": { + "name": "initiated_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "return_to": { + "name": "return_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_auth_states_expires_idx": { + "name": "oauth_auth_states_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_connections": { + "name": "oauth_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_id": { + "name": "external_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_email": { + "name": "external_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_account_name": { + "name": "external_account_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "access_token_ciphertext": { + "name": "access_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_iv": { + "name": "access_token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_tag": { + "name": "access_token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token_ciphertext": { + "name": "refresh_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_iv": { + "name": "refresh_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_tag": { + "name": "refresh_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_connections_org_provider_idx": { + "name": "oauth_connections_org_provider_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_connections_org_idx": { + "name": "oauth_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_onboarding_state": { + "name": "org_onboarding_state", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_data_requested": { + "name": "demo_data_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "checklist_dismissed_at": { + "name": "checklist_dismissed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "first_data_received_at": { + "name": "first_data_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "welcome_email_sent_at": { + "name": "welcome_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "connect_nudge_email_sent_at": { + "name": "connect_nudge_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "stalled_email_sent_at": { + "name": "stalled_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "activation_email_sent_at": { + "name": "activation_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_attribute_mappings": { + "name": "org_ingest_attribute_mappings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_context": { + "name": "source_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_key": { + "name": "target_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_ingest_attribute_mappings_org_idx": { + "name": "org_ingest_attribute_mappings_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_recommendation_issues": { + "name": "org_recommendation_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "recommendation_key": { + "name": "recommendation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_key": { + "name": "canonical_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "usage_count": { + "name": "usage_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "org_recommendation_issues_org_idx": { + "name": "org_recommendation_issues_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_recommendation_issues_org_key_idx": { + "name": "org_recommendation_issues_org_key_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "recommendation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_keys": { + "name": "org_ingest_keys", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key_hash": { + "name": "public_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_ciphertext": { + "name": "private_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_iv": { + "name": "private_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_tag": { + "name": "private_key_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_hash": { + "name": "private_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_rotated_at": { + "name": "public_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "private_rotated_at": { + "name": "private_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "org_ingest_keys_public_key_unique": { + "name": "org_ingest_keys_public_key_unique", + "columns": [ + { + "expression": "public_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_public_key_hash_unique": { + "name": "org_ingest_keys_public_key_hash_unique", + "columns": [ + { + "expression": "public_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_private_key_hash_unique": { + "name": "org_ingest_keys_private_key_hash_unique", + "columns": [ + { + "expression": "private_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_ingest_keys_org_id_pk": { + "name": "org_ingest_keys_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_sampling_policies": { + "name": "org_ingest_sampling_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "trace_sample_ratio": { + "name": "trace_sample_ratio", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "always_keep_error_spans": { + "name": "always_keep_error_spans", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "always_keep_slow_spans_ms": { + "name": "always_keep_slow_spans_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_settings": { + "name": "org_clickhouse_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_url": { + "name": "ch_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_user": { + "name": "ch_user", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_password_ciphertext": { + "name": "ch_password_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_iv": { + "name": "ch_password_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_tag": { + "name": "ch_password_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_database": { + "name": "ch_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema_version": { + "name": "schema_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_settings_org_id_pk": { + "name": "org_clickhouse_settings_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_schema_apply_runs": { + "name": "org_clickhouse_schema_apply_runs", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_migration": { + "name": "current_migration", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_total": { + "name": "steps_total", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_done": { + "name": "steps_done", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "applied_versions": { + "name": "applied_versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "skipped": { + "name": "skipped", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_schema_apply_runs_org_id_pk": { + "name": "org_clickhouse_schema_apply_runs_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_connections": { + "name": "planetscale_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ps_organization": { + "name": "ps_organization", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scrape_target_id": { + "name": "scrape_target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_ciphertext": { + "name": "webhook_secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_iv": { + "name": "webhook_secret_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_tag": { + "name": "webhook_secret_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detected_permissions_json": { + "name": "detected_permissions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_inventory_at": { + "name": "last_inventory_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_inventory_error": { + "name": "last_inventory_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_connections_org_idx": { + "name": "planetscale_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_databases": { + "name": "planetscale_databases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mysql'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "branches_json": { + "name": "branches_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_databases_org_db_idx": { + "name": "planetscale_databases_org_db_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_databases_org_idx": { + "name": "planetscale_databases_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_events": { + "name": "planetscale_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "database_name": { + "name": "database_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_login": { + "name": "actor_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_events_dedupe_idx": { + "name": "planetscale_events_dedupe_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_events_org_db_time_idx": { + "name": "planetscale_events_org_db_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_events_org_time_idx": { + "name": "planetscale_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_poll_state": { + "name": "planetscale_poll_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_poll_state_org_dataset_db_idx": { + "name": "planetscale_poll_state_org_dataset_db_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_poll_state_org_idx": { + "name": "planetscale_poll_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_target_checks": { + "name": "scrape_target_checks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "byDefault", + "name": "scrape_target_checks_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_target_key": { + "name": "sub_target_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_scraped": { + "name": "samples_scraped", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_post_relabel": { + "name": "samples_post_relabel", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "scrape_target_checks_target_checked_idx": { + "name": "scrape_target_checks_target_checked_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scrape_target_checks_target_id_scrape_targets_id_fk": { + "name": "scrape_target_checks_target_id_scrape_targets_id_fk", + "tableFrom": "scrape_target_checks", + "tableTo": "scrape_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_targets": { + "name": "scrape_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prometheus'" + }, + "discovery_config_json": { + "name": "discovery_config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scrape_interval_seconds": { + "name": "scrape_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 15 + }, + "labels_json": { + "name": "labels_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "managed_by": { + "name": "managed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_ciphertext": { + "name": "auth_credentials_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_iv": { + "name": "auth_credentials_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_tag": { + "name": "auth_credentials_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_scrape_at": { + "name": "last_scrape_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_scrape_error": { + "name": "last_scrape_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "scrape_targets_org_idx": { + "name": "scrape_targets_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scrape_targets_org_enabled_idx": { + "name": "scrape_targets_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_workspaces": { + "name": "slack_workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_name": { + "name": "team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_user_id": { + "name": "bot_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_ciphertext": { + "name": "bot_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_iv": { + "name": "bot_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_tag": { + "name": "bot_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_ciphertext": { + "name": "api_key_secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_iv": { + "name": "api_key_secret_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_tag": { + "name": "api_key_secret_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_reason": { + "name": "revoked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "slack_workspaces_team_id_idx": { + "name": "slack_workspaces_team_id_idx", + "columns": [ + { + "expression": "team_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_workspaces_org_idx": { + "name": "slack_workspaces_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_workspaces_active_org_idx": { + "name": "slack_workspaces_active_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"slack_workspaces\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_commits": { + "name": "vcs_commits", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_email": { + "name": "author_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_avatar_url": { + "name": "author_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authored_at": { + "name": "authored_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "committed_at": { + "name": "committed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_commits_repo_sha_idx": { + "name": "vcs_commits_repo_sha_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_commits_org_sha_idx": { + "name": "vcs_commits_org_sha_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_installations": { + "name": "vcs_installations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_installation_id": { + "name": "external_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_account_id": { + "name": "external_account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_avatar_url": { + "name": "account_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repository_selection": { + "name": "repository_selection", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_installations_provider_external_idx": { + "name": "vcs_installations_provider_external_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_installations_org_idx": { + "name": "vcs_installations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repositories": { + "name": "vcs_repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "tracked_branch": { + "name": "tracked_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_archived": { + "name": "is_archived", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repositories_org_repo_idx": { + "name": "vcs_repositories_org_repo_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_repo_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_org_idx": { + "name": "vcs_repositories_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_installation_idx": { + "name": "vcs_repositories_installation_idx", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repository_branches": { + "name": "vcs_repository_branches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repository_branches_repo_name_idx": { + "name": "vcs_repository_branches_repo_name_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repository_branches_org_idx": { + "name": "vcs_repository_branches_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 826446dba..477f454e6 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -344,6 +344,13 @@ "when": 1787920777688, "tag": "0049_home_list_indexes", "breakpoints": true + }, + { + "idx": 49, + "version": "7", + "when": 1788007454701, + "tag": "0050_audit_log_entries", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/packages/db/src/schema/audit-log.ts b/packages/db/src/schema/audit-log.ts new file mode 100644 index 000000000..c3990b67e --- /dev/null +++ b/packages/db/src/schema/audit-log.ts @@ -0,0 +1,61 @@ +import { index, jsonb, pgTable, primaryKey, text, timestamp } from "drizzle-orm/pg-core" +import type { ActorId, ApiKeyId, AuditLogEntryId, OrgId, UserId } from "@maple/domain/primitives" +import type { AuditActorType, AuditLogSource, AuditOutcome } from "@maple/domain/http" + +/** + * Append-only org-wide audit trail: every allowed or denied action an + * identified actor performs against Maple, whether it arrived from the + * dashboard, the public API, or MCP. `userId`/`apiKeyId`/`actorId` identify the + * credential-holder per `actorType` — for `agent` rows `userId` is the human + * the agent acted on behalf of. `actorLabel` freezes a display name at write + * time so entries stay readable after keys are rolled or agents renamed. + * Rows arrive through the audit events queue; `occurredAt` is stamped by the + * producer, `recordedAt` by the consumer at insert. + */ +export const auditLogEntries = pgTable( + "audit_log_entries", + { + orgId: text("org_id").$type().notNull(), + id: text("id").$type().notNull(), + actorType: text("actor_type").$type().notNull(), + userId: text("user_id").$type(), + apiKeyId: text("api_key_id").$type(), + actorId: text("actor_id").$type(), + actorLabel: text("actor_label"), + affectedUserId: text("affected_user_id").$type(), + source: text("source").$type().notNull(), + action: text("action").notNull(), + outcome: text("outcome").$type().notNull(), + denialReason: text("denial_reason"), + resourceType: text("resource_type"), + resourceId: text("resource_id"), + // Field names touched by an update, queryable without parsing changesJson. + changedFields: text("changed_fields").array(), + changesJson: jsonb("changes_json").$type(), + metadataJson: jsonb("metadata_json").$type(), + requestId: text("request_id"), + originIp: text("origin_ip"), + originCountry: text("origin_country"), + occurredAt: timestamp("occurred_at", { withTimezone: true, mode: "date" }).notNull(), + recordedAt: timestamp("recorded_at", { withTimezone: true, mode: "date" }).notNull(), + }, + (table) => [ + primaryKey({ columns: [table.orgId, table.id] }), + index("audit_log_entries_org_occurred_idx").on(table.orgId, table.occurredAt), + index("audit_log_entries_org_actor_type_occurred_idx").on( + table.orgId, + table.actorType, + table.occurredAt, + ), + index("audit_log_entries_org_resource_idx").on(table.orgId, table.resourceType, table.resourceId), + index("audit_log_entries_org_request_idx").on(table.orgId, table.requestId), + index("audit_log_entries_org_outcome_occurred_idx").on( + table.orgId, + table.outcome, + table.occurredAt, + ), + ], +) + +export type AuditLogEntryRow = typeof auditLogEntries.$inferSelect +export type AuditLogEntryInsert = typeof auditLogEntries.$inferInsert diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 9699888fd..93181338c 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -2,6 +2,7 @@ export * from "./ai-triage" export * from "./alerts" export * from "./anomalies" export * from "./api-keys" +export * from "./audit-log" export * from "./cloudflare-analytics-state" export * from "./cloudflare-hyperdrive-configs" export * from "./cloudflare-logpush-connectors" diff --git a/packages/domain/src/http/audit-log.ts b/packages/domain/src/http/audit-log.ts new file mode 100644 index 000000000..19d572532 --- /dev/null +++ b/packages/domain/src/http/audit-log.ts @@ -0,0 +1,54 @@ +import { Schema } from "effect" +import { HttpTaggedError } from "./error-policy" + +/** + * Who performed an audited action. `user` is a dashboard session, `api_key` a + * v1/v2 public-API credential, `agent` a registered LLM agent acting over MCP, + * and `system` Maple itself (crons, sweeps, lifecycle automation). + */ +export const AuditActorType = Schema.Literals(["user", "api_key", "agent", "system"]).annotate({ + identifier: "@maple/AuditActorType", + title: "Audit Actor Type", +}) +export type AuditActorType = Schema.Schema.Type + +/** Which surface the audited request arrived through. */ +export const AuditLogSource = Schema.Literals(["dashboard", "api", "mcp", "system"]).annotate({ + identifier: "@maple/AuditLogSource", + title: "Audit Log Source", +}) +export type AuditLogSource = Schema.Schema.Type + +/** Whether the action was performed or refused — denied attempts are logged too. */ +export const AuditOutcome = Schema.Literals(["allowed", "denied"]).annotate({ + identifier: "@maple/AuditOutcome", + title: "Audit Outcome", +}) +export type AuditOutcome = Schema.Schema.Type + +/** Before/after diff of an update, with the touched field names queryable on their own. */ +export const AuditChanges = Schema.Struct({ + fields: Schema.Array(Schema.String), + before: Schema.Record(Schema.String, Schema.Unknown), + after: Schema.Record(Schema.String, Schema.Unknown), +}).annotate({ + identifier: "@maple/AuditChanges", + title: "Audit Changes", +}) +export type AuditChanges = Schema.Schema.Type + +export class AuditLogPersistenceError extends HttpTaggedError()( + "@maple/http/errors/AuditLogPersistenceError", + { + message: Schema.String, + }, + { + status: 503, + code: "audit_log_unavailable", + title: "The audit log is temporarily unavailable", + message: "The audit log is temporarily unavailable. Retry in a few seconds.", + retry: "backoff", + recovery: "retry", + exposure: "redacted", + }, +) {} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 9bcacd800..5d9d6e87f 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -5,6 +5,7 @@ export * from "./ai-triage" export * from "./investigations" export * from "./anomalies" export * from "./api-keys" +export * from "./audit-log" export * from "./alerts" export * from "./mobile-devices" export * from "./auth" diff --git a/packages/domain/src/http/v2/api.ts b/packages/domain/src/http/v2/api.ts index 53ec8e161..e0814e060 100644 --- a/packages/domain/src/http/v2/api.ts +++ b/packages/domain/src/http/v2/api.ts @@ -6,6 +6,7 @@ import { V2AlertIncidentsApiGroup } from "./alert-incidents" import { V2AlertRulesApiGroup } from "./alert-rules" import { V2ApiKeysApiGroup } from "./api-keys" import { V2AttributeMappingsApiGroup } from "./attribute-mappings" +import { V2AuditLogApiGroup } from "./audit-log" import { V2DashboardsApiGroup } from "./dashboards" import { V2IngestKeysApiGroup } from "./ingest-keys" import { V2SlackIntegrationsApiGroup } from "./integrations" @@ -94,6 +95,7 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") .add(V2PlanetScaleIntegrationsApiGroup) .add(V2ErrorIssuesApiGroup) .add(V2AttributeMappingsApiGroup) + .add(V2AuditLogApiGroup) .add(V2ScrapeTargetsApiGroup) .add(V2InstrumentationRecommendationsApiGroup) .add(V2InstrumentationAuditApiGroup) diff --git a/packages/domain/src/http/v2/audit-log.ts b/packages/domain/src/http/v2/audit-log.ts new file mode 100644 index 000000000..74bae4dc4 --- /dev/null +++ b/packages/domain/src/http/v2/audit-log.ts @@ -0,0 +1,246 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" +import { Schema } from "effect" +import { AuditLogEntryId } from "../../primitives" +import { + AuditActorType, + AuditLogPersistenceError, + AuditLogSource, + AuditOutcome, +} from "../audit-log" +import { AuthorizationV2 } from "./auth" +import { wireExample, ListOf, ListQuery, Timestamp } from "./envelopes" +import { V2ParameterInvalid } from "./errors" +import { publicErrors } from "./public-error" +import { PublicId, PublicIdPrefixes } from "./public-id" + +/** `alog_…` public ID ⇄ internal `AuditLogEntryId` (raw UUID). */ +export const AuditLogEntryPublicId = PublicId(PublicIdPrefixes.auditLogEntry, AuditLogEntryId) + +const actorTypeField = AuditActorType.annotate({ + description: + "Who performed the action: `user` (a dashboard session), `api_key` (a public-API credential), `agent` (a registered LLM agent acting over MCP), or `system` (Maple automation).", + examples: ["user"], +}) + +const sourceField = AuditLogSource.annotate({ + description: + "The surface the request arrived through: `dashboard`, `api` (the public v1/v2 API), `mcp`, or `system`.", + examples: ["dashboard"], +}) + +const outcomeField = AuditOutcome.annotate({ + description: + "Whether the action was performed (`allowed`) or refused (`denied`). Denied attempts — e.g. an API key lacking the required scope — are logged too.", + examples: ["allowed"], +}) + +export const V2AuditChanges = Schema.Struct({ + fields: Schema.Array(Schema.String).annotate({ + description: "Names of the fields the update touched.", + examples: [["name"]], + }), + before: Schema.Record(Schema.String, Schema.Unknown).annotate({ + description: "Prior values of the touched fields.", + }), + after: Schema.Record(Schema.String, Schema.Unknown).annotate({ + description: "New values of the touched fields.", + }), +}).annotate({ + identifier: "AuditLogChanges", + title: "Audit Log Changes", + description: "The before/after diff an update applied, keyed by field name.", +}) +export type V2AuditChanges = Schema.Schema.Type + +const auditLogEntryExample = { + id: "alog_4CzLmR1pTxWvYbNhQd82Kf", + object: "audit_log_entry", + action: "alert_rule.updated", + outcome: "allowed", + denial_reason: null, + actor_type: "user", + actor_id: "user_2fj3K9dLqWm8xYbT", + actor_name: "David", + affected_user: null, + source: "dashboard", + resource_type: "alert_rule", + resource_id: "alrt_YofPTrK9782DWwcnXhpcCw", + changes: { fields: ["name"], before: { name: "Errors" }, after: { name: "High error rate" } }, + metadata: null, + request_id: "8f2c1a9d4b7e3f60", + origin_ip: "203.0.113.7", + origin_country: "DE", + occurred_at: "2026-08-29T09:12:00.000Z", + recorded_at: "2026-08-29T09:12:00.412Z", +} as const + +// v2 wire schemas are annotated `Schema.Struct`s (not `Schema.Class`) — see the +// note in api-keys.ts. +export const V2AuditLogEntry = Schema.Struct({ + id: AuditLogEntryPublicId, + object: Schema.Literal("audit_log_entry").annotate({ + description: 'The object type — always `"audit_log_entry"`.', + examples: ["audit_log_entry"], + }), + action: Schema.String.annotate({ + description: "What happened, as `.` (e.g. `alert_rule.created`, `api_key.rolled`).", + examples: ["alert_rule.created"], + }), + outcome: outcomeField, + denial_reason: Schema.NullOr(Schema.String).annotate({ + description: "Why the action was refused, when `outcome` is `denied`; otherwise `null`.", + }), + actor_type: actorTypeField, + actor_id: Schema.NullOr(Schema.String).annotate({ + description: + "Public identifier of the actor: a `user_…` ID for users, a `key_…` ID for API keys, an `actor_…` ID for agents, or `null` for system actions.", + examples: ["user_2fj3K9dLqWm8xYbT"], + }), + actor_name: Schema.NullOr(Schema.String).annotate({ + description: + "Display name of the actor at the time of the action (agent name, API key name, …), or `null` when none was recorded.", + examples: ["David"], + }), + affected_user: Schema.NullOr(Schema.String).annotate({ + description: + "The `user_…` ID of the user the action was performed on (e.g. a removed member), when different from the actor; otherwise `null`.", + }), + source: sourceField, + resource_type: Schema.NullOr(Schema.String).annotate({ + description: "The kind of resource acted on (e.g. `alert_rule`, `dashboard`), or `null`.", + examples: ["alert_rule"], + }), + resource_id: Schema.NullOr(Schema.String).annotate({ + description: "Public ID of the resource acted on, or `null`.", + examples: ["alrt_YofPTrK9782DWwcnXhpcCw"], + }), + changes: Schema.NullOr(V2AuditChanges).annotate({ + description: "The before/after diff for updates, or `null` when the action carries no diff.", + }), + metadata: Schema.NullOr(Schema.Record(Schema.String, Schema.Unknown)).annotate({ + description: "Action-specific context recorded with the entry, or `null`.", + }), + request_id: Schema.NullOr(Schema.String).annotate({ + description: + "Identifier of the HTTP request that performed the action, shared by every entry the request produced; or `null`.", + }), + origin_ip: Schema.NullOr(Schema.String).annotate({ + description: "Client IP the request originated from, or `null`.", + }), + origin_country: Schema.NullOr(Schema.String).annotate({ + description: "ISO 3166-1 country code the request originated from, or `null`.", + }), + occurred_at: Timestamp.annotate({ description: "When the action happened." }), + recorded_at: Timestamp.annotate({ + description: + "When the entry was durably recorded. Trails `occurred_at` by the audit pipeline's delivery latency.", + }), +}).annotate({ + identifier: "AuditLogEntry", + title: "Audit Log Entry", + description: + "One entry in the organization's append-only audit log: an allowed or denied action performed by a user, API key, or agent against a Maple resource.", + examples: [wireExample(auditLogEntryExample)], +}) +export type V2AuditLogEntry = Schema.Schema.Type + +/** Audit-log list query: standard pagination plus actor/action/resource/outcome/time filters. */ +export const V2AuditLogQuery = Schema.Struct({ + ...ListQuery.fields, + actor_type: Schema.optional( + AuditActorType.annotate({ + description: "Only return entries performed by this kind of actor.", + }), + ), + actor_id: Schema.optional( + Schema.String.annotate({ + description: + "Only return entries performed by this specific actor: a `user_…` user ID, `key_…` API key ID, or `actor_…` agent ID.", + }), + ), + affected_user: Schema.optional( + Schema.String.annotate({ + description: "Only return entries that acted on this `user_…` user.", + }), + ), + action: Schema.optional( + Schema.String.annotate({ + description: "Only return entries with exactly this action (e.g. `alert_rule.created`).", + }), + ), + outcome: Schema.optional( + AuditOutcome.annotate({ + description: "Only return entries with this outcome.", + }), + ), + resource_type: Schema.optional( + Schema.String.annotate({ + description: "Only return entries acting on this kind of resource (e.g. `dashboard`).", + }), + ), + resource_id: Schema.optional( + Schema.String.annotate({ + description: + "Only return entries acting on this exact resource, by its public ID (e.g. `dash_…`).", + }), + ), + changed: Schema.optional( + Schema.String.annotate({ + description: "Only return entries whose update touched this field name (e.g. `scopes`).", + }), + ), + request_id: Schema.optional( + Schema.String.annotate({ + description: "Only return entries produced by this HTTP request.", + }), + ), + since: Schema.optional( + Timestamp.annotate({ + description: "Only return entries that occurred at or after this time.", + }), + ), + until: Schema.optional( + Timestamp.annotate({ + description: "Only return entries that occurred at or before this time.", + }), + ), +}).annotate({ + identifier: "AuditLogQuery", + title: "Audit log query", + description: + "Pagination plus optional actor, action, outcome, resource, changed-field, request, and time-window filters.", +}) +export type V2AuditLogQuery = Schema.Schema.Type + +const [auditLogPersistence] = publicErrors(AuditLogPersistenceError) + +const AuditLogEntryList = ListOf(V2AuditLogEntry).annotate({ + identifier: "AuditLogEntryList", + title: "Audit log entry list", + description: "A cursor-paginated page of audit log entries, newest first.", +}) + +export class V2AuditLogApiGroup extends HttpApiGroup.make("auditLog") + .add( + HttpApiEndpoint.get("list", "/", { + query: V2AuditLogQuery, + success: AuditLogEntryList, + error: [V2ParameterInvalid.schema, auditLogPersistence], + }).annotateMerge( + OpenApi.annotations({ + identifier: "listAuditLogEntries", + summary: "List audit log entries", + description: + "Returns your organization's audit log, newest first, optionally filtered by actor, action, outcome, resource, changed field, request, and time window. Cursor-paginated. Requires the `audit_log:read` scope.", + }), + ), + ) + .prefix("/v2/audit_log") + .middleware(AuthorizationV2) + .annotateMerge( + OpenApi.annotations({ + title: "Audit Log", + description: + "The organization's append-only audit trail — every allowed or denied action performed through the dashboard, the public API, or MCP, attributed to the user, API key, or agent that performed it, with before/after diffs for updates.", + }), + ) {} diff --git a/packages/domain/src/http/v2/index.ts b/packages/domain/src/http/v2/index.ts index e8b7d3ab1..b04facc3b 100644 --- a/packages/domain/src/http/v2/index.ts +++ b/packages/domain/src/http/v2/index.ts @@ -6,6 +6,7 @@ export * from "./anomalies" export * from "./api" export * from "./api-keys" export * from "./attribute-mappings" +export * from "./audit-log" export * from "./auth" export * from "./dashboards" export * from "./envelopes" diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index 9d0dfa744..e0ee0573c 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -118,6 +118,7 @@ describe("MapleApiV2 OpenAPI", () => { "GET /v2/api_keys/{id}", "GET /v2/attribute_mappings", "GET /v2/attribute_mappings/{id}", + "GET /v2/audit_log", "GET /v2/dashboards", "GET /v2/dashboards/templates", "GET /v2/dashboards/{id}", diff --git a/packages/domain/src/http/v2/public-id.ts b/packages/domain/src/http/v2/public-id.ts index 6136f9d06..52f80200f 100644 --- a/packages/domain/src/http/v2/public-id.ts +++ b/packages/domain/src/http/v2/public-id.ts @@ -28,6 +28,7 @@ export const PublicIdPrefixes = { alertDestination: "dest", alertIncident: "inc", actor: "actor", + auditLogEntry: "alog", errorIssue: "iss", errorIncident: "einc", investigation: "inv", diff --git a/packages/primitives/src/index.ts b/packages/primitives/src/index.ts index 4b8467d80..3c909ff7e 100644 --- a/packages/primitives/src/index.ts +++ b/packages/primitives/src/index.ts @@ -128,6 +128,9 @@ export type ActorId = Schema.Schema.Type export const ErrorIssueEventId = MapleUuidId("@maple/ErrorIssueEventId", "Error Issue Event ID") export type ErrorIssueEventId = Schema.Schema.Type +export const AuditLogEntryId = MapleUuidId("@maple/AuditLogEntryId", "Audit Log Entry ID") +export type AuditLogEntryId = Schema.Schema.Type + export const ErrorIssuePullRequestId = MapleUuidId( "@maple/ErrorIssuePullRequestId", "Error Issue Pull Request ID", From b21b3a39a1fb93b7ac903935ef324ac724fdaf7f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 29 Aug 2026 15:09:41 +0200 Subject: [PATCH 02/19] fix(audit): satisfy effect-boundary lint and regenerate iOS OpenAPI spec - Tagged AuditQueueSendError instead of a global Error in the queue send failure channel. - compactAuditChanges takes static placeholder strings instead of unknown-typed summarizer functions; null still survives as null. - destinationObservableValue returns a concrete union, not unknown. - iOS OpenAPI spec regenerated for the new /v2/audit_log path. --- .../src/routes/v2/alert-destinations.http.ts | 5 ++++- apps/api/src/routes/v2/alert-rules.http.ts | 5 ++--- apps/api/src/routes/v2/audit-changes.ts | 19 ++++++++++--------- apps/api/src/routes/v2/dashboards.http.ts | 10 ++++------ .../api/src/services/audit/AuditLogService.ts | 11 ++++++++++- .../MapleAPI/Sources/MapleAPI/openapi.json | 4 ++++ 6 files changed, 34 insertions(+), 20 deletions(-) diff --git a/apps/api/src/routes/v2/alert-destinations.http.ts b/apps/api/src/routes/v2/alert-destinations.http.ts index 73238c668..4f586d98b 100644 --- a/apps/api/src/routes/v2/alert-destinations.http.ts +++ b/apps/api/src/routes/v2/alert-destinations.http.ts @@ -196,7 +196,10 @@ const toUpdateRequest = (params: V2AlertDestinationUpdateParams): AlertDestinati const destinationSecretKeys = new Set(["integrationKey", "signingSecret", "url", "webhookUrl", "botToken"]) /** Fields of an update that are readable back off the destination document. */ -const destinationObservableValue = (doc: AlertDestinationDocument, key: string): unknown => { +const destinationObservableValue = ( + doc: AlertDestinationDocument, + key: string, +): string | boolean | ReadonlyArray | null | undefined => { switch (key) { case "name": return doc.name diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 54c61591c..45e53cabf 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -132,8 +132,6 @@ const ruleAuditKeys: ReadonlyArray (value === null ? null : "") const toV2RuleMutationResponse = (doc: AlertRuleDocument): V2AlertRuleMutationResponse => ({ ...toV2Rule(doc), @@ -413,7 +411,8 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules pickPresentFields(ruleAuditKeys, payload, toV2Rule(current)), pickPresentFields(ruleAuditKeys, payload, toV2Rule(updated)), ), - { query_builder_draft: summarizeRuleBlob, raw_query_sql: summarizeRuleBlob }, + // Query drafts and raw SQL are config blobs — audit that they changed, not their bodies. + { query_builder_draft: "", raw_query_sql: "" }, ) yield* recordHttpAudit("alert_rule.updated", { resourceType: "alert_rule", diff --git a/apps/api/src/routes/v2/audit-changes.ts b/apps/api/src/routes/v2/audit-changes.ts index afa070f61..444a9fed4 100644 --- a/apps/api/src/routes/v2/audit-changes.ts +++ b/apps/api/src/routes/v2/audit-changes.ts @@ -40,21 +40,22 @@ export const pickPresentFields = ( } /** - * Replace selected fields' before/after values with a compact summary so large - * config blobs (dashboard widgets, query drafts) don't bloat the audit row. + * Replace selected fields' before/after values with a static placeholder so + * large config blobs (dashboard widgets, query drafts) and secrets don't reach + * the audit row. Null survives, so "cleared" still reads as cleared. */ export const compactAuditChanges = ( changes: AuditChanges | undefined, - summarize: Record unknown>, + placeholders: Record, ): AuditChanges | undefined => { if (changes === undefined) return undefined - const before: Record = { ...changes.before } - const after: Record = { ...changes.after } + const before = { ...changes.before } + const after = { ...changes.after } for (const field of changes.fields) { - const summary = summarize[field] - if (summary === undefined) continue - if (field in before) before[field] = summary(before[field]) - if (field in after) after[field] = summary(after[field]) + const placeholder = placeholders[field] + if (placeholder === undefined) continue + if (field in before && before[field] !== null) before[field] = placeholder + if (field in after && after[field] !== null) after[field] = placeholder } return { fields: changes.fields, before, after } } diff --git a/apps/api/src/routes/v2/dashboards.http.ts b/apps/api/src/routes/v2/dashboards.http.ts index 604285f25..a1c0ef931 100644 --- a/apps/api/src/routes/v2/dashboards.http.ts +++ b/apps/api/src/routes/v2/dashboards.http.ts @@ -190,9 +190,6 @@ const dashboardAuditKeys: ReadonlyArray (value: unknown) => - Array.isArray(value) ? `<${value.length} ${label}>` : "" const encodeVersionCursor = (versionNumber: number): string => `ver_${versionNumber.toString(36)}` @@ -410,10 +407,11 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards pickPresentFields(dashboardAuditKeys, payload, toV2Dashboard(previous)), pickPresentFields(dashboardAuditKeys, payload, toV2Dashboard(dashboard)), ), + // Layout arrays are config blobs — audit that they changed, not their bodies. { - widgets: summarizeListBlob("widgets"), - sections: summarizeListBlob("sections"), - variables: summarizeListBlob("variables"), + widgets: "", + sections: "", + variables: "", }, ) yield* recordHttpAudit("dashboard.updated", { diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index 4472dcb68..16a514cef 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -16,6 +16,14 @@ import { AuditLogEvent, auditEventToInsert, encodeAuditLogEventSync } from "./au const decodeAuditLogEntryIdSync = Schema.decodeUnknownSync(AuditLogEntryIdSchema) +class AuditQueueSendError extends Schema.TaggedError()( + "@maple/api/services/audit/AuditQueueSendError", + { + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + }, +) {} + /** Producer binding name; the paired `*_NAME` var drives consumer dispatch. */ export const AUDIT_EVENTS_QUEUE_BINDING = "AUDIT_EVENTS_QUEUE" @@ -118,7 +126,8 @@ export class AuditLogService extends Context.Service queue.send(encodeAuditLogEventSync(event)), - catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), + catch: (cause) => + new AuditQueueSendError({ message: "Audit queue send failed", cause }), }).pipe( // Queue unavailability must not lose the entry: degrade to a // direct write before giving up. diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index 7b31b3aa8..f1155e639 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -7335,6 +7335,10 @@ "description": "Ingest-time attribute rewrite rules. Move or copy span/resource attribute values to new keys as telemetry arrives, normalizing naming across services without redeploying them.", "name": "Attribute Mappings" }, + { + "description": "The organization's append-only audit trail — every allowed or denied action performed through the dashboard, the public API, or MCP, attributed to the user, API key, or agent that performed it, with before/after diffs for updates.", + "name": "Audit Log" + }, { "description": "Metrics endpoints Maple scrapes on a schedule — self-hosted Prometheus endpoints and PlanetScale branch metrics. Manage targets, probe them on demand, and inspect recent scrape checks. Credentials are write-only.", "name": "Scrape Targets" From 416ee4c20d4db0f148ef4b495f001cc61185b415 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 29 Aug 2026 17:28:38 +0200 Subject: [PATCH 03/19] =?UTF-8?q?fix(audit):=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20admin=20gate,=20denial=20coalescing,=20diff=20safety,=20inde?= =?UTF-8?q?xes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From four review passes over the audit log: - Admin-gate GET /v2/audit_log: entries carry every member's activity, denial history and origin IP for the retention window. The settings tab hides for non-admins to match. - Coalesce denied api.request records per (org, key, method+path, reason) in a 60s isolate-local window. The v1 auth layer has no rate limiter, so a client looping mis-scoped requests could otherwise amplify into unbounded queue messages, rows and warn logs. Both auth layers now share one helper, so v1 records the same forensics as v2. - Audit the two v2 denial branches that returned early (MCP-only key, invalid device credential) — the credential-probing case the feature exists to surface. - Bound queue.send with a 2s timeout: a stalling broker must not hang the mutation's response before the direct-write fallback. - Replace blanket catchCause with catchTag/catchDefect so interruption propagates instead of spawning a Postgres insert mid-teardown. - Structural (key-order insensitive) diff comparison; redact userinfo and query strings from audited scrape-target URLs; carry the cause on AuditLogPersistenceError. - Index occurred_at for the retention sweep, the actor-identity columns for the primary 'what did this credential do' query, and a GIN index for changed-field lookups. --- apps/api/src/routes/v2/alert-rules.http.ts | 4 +- apps/api/src/routes/v2/audit-changes.ts | 32 +++- apps/api/src/routes/v2/audit-log.http.ts | 8 + apps/api/src/routes/v2/dashboards.http.ts | 36 +++- apps/api/src/routes/v2/scrape-targets.http.ts | 26 ++- .../api/src/services/audit/AuditLogService.ts | 48 +++-- .../src/services/audit/audit-log-retention.ts | 19 +- .../services/auth/ApiAuthorizationLayer.ts | 16 +- .../services/auth/ApiAuthorizationV2Layer.ts | 51 ++---- apps/api/src/services/auth/audit-denial.ts | 94 ++++++++++ .../src/components/settings/settings-nav.tsx | 3 + .../db/drizzle/0050_audit_log_entries.sql | 9 +- packages/db/drizzle/meta/0050_snapshot.json | 167 +++++++++++++++++- packages/db/src/schema/audit-log.ts | 19 ++ packages/domain/src/http/audit-log.ts | 2 + packages/domain/src/http/v2/audit-log.ts | 8 +- 16 files changed, 466 insertions(+), 76 deletions(-) create mode 100644 apps/api/src/services/auth/audit-denial.ts diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index 45e53cabf..fe1f5122d 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -412,7 +412,9 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules pickPresentFields(ruleAuditKeys, payload, toV2Rule(updated)), ), // Query drafts and raw SQL are config blobs — audit that they changed, not their bodies. - { query_builder_draft: "", raw_query_sql: "" }, + { query_builder_draft: "", raw_query_sql: "" } satisfies Partial< + Record<(typeof ruleAuditKeys)[number], string> + >, ) yield* recordHttpAudit("alert_rule.updated", { resourceType: "alert_rule", diff --git a/apps/api/src/routes/v2/audit-changes.ts b/apps/api/src/routes/v2/audit-changes.ts index 444a9fed4..40853fd03 100644 --- a/apps/api/src/routes/v2/audit-changes.ts +++ b/apps/api/src/routes/v2/audit-changes.ts @@ -1,5 +1,23 @@ import type { AuditChanges } from "@maple/domain/http" +/** + * Structural equality, insensitive to object key order (a server-rebuilt + * `timeRange` must not diff against the decoded payload echo). Arrays stay + * order-sensitive; anything non-JSON-shaped falls back to reference equality. + */ +export const structuralEqual = (a: unknown, b: unknown): boolean => { + if (a === b) return true + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false + return a.every((item, index) => structuralEqual(item, b[index])) + } + const aEntries = Object.entries(a) + const bEntries = new Map(Object.entries(b)) + if (aEntries.length !== bEntries.size) return false + return aEntries.every(([key, value]) => bEntries.has(key) && structuralEqual(value, bEntries.get(key))) +} + /** * Diff two snapshots restricted to the keys of `after` (the fields the request * actually touched — omitted fields are unchanged by contract). Returns @@ -15,7 +33,7 @@ export const diffAuditChanges = ( for (const key of Object.keys(after)) { const prev = before[key] const next = after[key] - if (JSON.stringify(prev) === JSON.stringify(next)) continue + if (structuralEqual(prev, next)) continue fields.push(key) beforeOut[key] = prev afterOut[key] = next @@ -46,6 +64,8 @@ export const pickPresentFields = ( */ export const compactAuditChanges = ( changes: AuditChanges | undefined, + // Call sites `satisfies Partial>` + // so a wire-key rename cannot silently disable a redaction placeholder. placeholders: Record, ): AuditChanges | undefined => { if (changes === undefined) return undefined @@ -59,3 +79,13 @@ export const compactAuditChanges = ( } return { fields: changes.fields, before, after } } + +/** + * Strip userinfo, query string, and fragment from a URL destined for an audit + * row — scrape URLs routinely embed tokens there. Keeps scheme/host/path. + */ +export const redactAuditUrl = (raw: string): string => { + if (!URL.canParse(raw)) return "" + const url = new URL(raw) + return `${url.protocol}//${url.host}${url.pathname}` +} diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts index a1a6e6e02..ec9b351a7 100644 --- a/apps/api/src/routes/v2/audit-log.http.ts +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -8,14 +8,18 @@ import { paginateOffsetQuery, PublicIdPrefixes, timestamp, + V2InsufficientPermissions, V2ParameterInvalid, } from "@maple/domain/http/v2" import type { V2AuditLogEntry } from "@maple/domain/http/v2" import type { AuditLogEntryRow } from "@maple/db" import { Effect, Option, Schema } from "effect" import { AuditLogService } from "@/services/audit/AuditLogService" +import { requireAdmin } from "@/services/auth/auth" import type { AuditLogListFilters } from "@/services/audit/AuditLogService" +const adminOnly = () => V2InsufficientPermissions.make("Only org admins can read the audit log") + const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) const decodeUserIdOption = Schema.decodeUnknownOption(UserId) @@ -99,6 +103,10 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( return handlers.handle("list", ({ query }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context + // The log carries every member's activity, denial history, and origin + // IP for the whole retention window — org admins only. Scoped API keys + // are additionally gated by `audit_log:read`. + yield* requireAdmin(tenant.roles, adminOnly) const identity = query.actor_id !== undefined ? yield* actorIdentityFilter(query.actor_id) : undefined const affectedUser = diff --git a/apps/api/src/routes/v2/dashboards.http.ts b/apps/api/src/routes/v2/dashboards.http.ts index a1c0ef931..6ba9b87c0 100644 --- a/apps/api/src/routes/v2/dashboards.http.ts +++ b/apps/api/src/routes/v2/dashboards.http.ts @@ -312,6 +312,15 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards ) yield* logShare("dashboard share rotated", context, { "maple.share.id": rotated.id }) + // Security event: rotation invalidates the previous public share token. + yield* recordHttpAudit("dashboard_share.rotated", { + resourceType: "dashboard_share", + resourceId: encodePublicId(PublicIdPrefixes.dashboardShare, rotated.id), + metadata: { + dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, dashboardId), + ...(widgetId === null ? undefined : { widget_id: widgetId }), + }, + }) return toV2DashboardShare(rotated) }) @@ -412,7 +421,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards widgets: "", sections: "", variables: "", - }, + } satisfies Partial>, ) yield* recordHttpAudit("dashboard.updated", { resourceType: "dashboard", @@ -450,6 +459,11 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards tenant.userId, converted.dashboard, ) + yield* recordHttpAudit("dashboard.created", { + resourceType: "dashboard", + resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + metadata: { name: dashboard.name, source: "perses_import" }, + }) return { object: "dashboard_import" as const, @@ -508,6 +522,17 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards params.id, params.version_id, ) + yield* recordHttpAudit("dashboard.version_restored", { + resourceType: "dashboard", + resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + metadata: { + name: dashboard.name, + version_id: encodePublicId( + PublicIdPrefixes.dashboardVersion, + params.version_id, + ), + }, + }) return toV2DashboardMutation(dashboard) }), @@ -600,6 +625,15 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards }) const tenant = yield* CurrentTenant.Context const dashboard = yield* persistence.create(tenant.orgId, tenant.userId, portable) + yield* recordHttpAudit("dashboard.created", { + resourceType: "dashboard", + resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + metadata: { + name: dashboard.name, + source: "template", + template_id: params.template_id, + }, + }) return toV2DashboardMutation(dashboard) }), diff --git a/apps/api/src/routes/v2/scrape-targets.http.ts b/apps/api/src/routes/v2/scrape-targets.http.ts index 0c9d1651b..71fee0fd6 100644 --- a/apps/api/src/routes/v2/scrape-targets.http.ts +++ b/apps/api/src/routes/v2/scrape-targets.http.ts @@ -11,7 +11,7 @@ import { } from "@maple/domain/http/v2" import type { V2ScrapeTarget, V2ScrapeTargetCheck } from "@maple/domain/http/v2" import { Effect } from "effect" -import { diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { diffAuditChanges, pickPresentFields, redactAuditUrl } from "@/routes/v2/audit-changes" import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ScrapeTargetsService } from "@/services/integrations/ScrapeTargetsService" @@ -185,10 +185,32 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT }), ) - const observable = diffAuditChanges( + // Read-then-write with no CAS: a concurrent update can make `before` + // reflect a state this update never saw. Accepted for audit purposes. + const diffed = diffAuditChanges( pickPresentFields(targetAuditKeys, payload, toV2ScrapeTarget(current)), pickPresentFields(targetAuditKeys, payload, toV2ScrapeTarget(updated)), ) + // Scrape URLs may carry tokens in userinfo/query — audit only scheme/host/path. + // Identical redacted values still mean the URL changed within the stripped part. + const observable = + diffed === undefined || !diffed.fields.includes("url") + ? diffed + : { + fields: diffed.fields, + before: { + ...diffed.before, + ...(typeof diffed.before["url"] === "string" + ? { url: redactAuditUrl(diffed.before["url"]) } + : undefined), + }, + after: { + ...diffed.after, + ...(typeof diffed.after["url"] === "string" + ? { url: redactAuditUrl(diffed.after["url"]) } + : undefined), + }, + } // Credentials are write-only: audit that they rotated, never their value. const changes = payload.auth_credentials !== undefined diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index 16a514cef..fcf2cf5ce 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -9,7 +9,7 @@ import { and, arrayContains, desc, eq, gte, lte } from "drizzle-orm" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import type { Queue } from "@cloudflare/workers-types" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" -import { Database } from "@/platform/DatabaseLive" +import { Database, type DatabaseError } from "@/platform/DatabaseLive" import { msToDate } from "@/platform/time" import { CurrentAuditActor } from "@/services/auth/audit-actor" import { AuditLogEvent, auditEventToInsert, encodeAuditLogEventSync } from "./audit-event" @@ -27,10 +27,15 @@ class AuditQueueSendError extends Schema.TaggedError()( /** Producer binding name; the paired `*_NAME` var drives consumer dispatch. */ export const AUDIT_EVENTS_QUEUE_BINDING = "AUDIT_EVENTS_QUEUE" -const toPersistenceError = (error: unknown) => - new AuditLogPersistenceError({ - message: error instanceof Error ? error.message : "Audit log query failed", - }) +/** + * `queue.send` sits on the response path of every mutation and denial. A + * healthy send is tens of ms; 2s bounds a stalling broker before the entry + * degrades to a direct Postgres write. + */ +export const AUDIT_QUEUE_SEND_TIMEOUT = "2 seconds" + +const toPersistenceError = (error: DatabaseError) => + new AuditLogPersistenceError({ message: error.message, cause: error }) /** The credential-holder behind an audited action, as known at the call site. */ export interface AuditActorRef { @@ -121,6 +126,14 @@ export class AuditLogService extends Context.Service + Effect.logWarning("Audit queue send failed; writing directly", { cause: error }).pipe( + Effect.andThen(insertDirect(event)), + ) + const publish = (event: AuditLogEvent) => queue === undefined ? insertDirect(event) @@ -129,12 +142,15 @@ export class AuditLogService extends Context.Service new AuditQueueSendError({ message: "Audit queue send failed", cause }), }).pipe( - // Queue unavailability must not lose the entry: degrade to a - // direct write before giving up. - Effect.catchCause((cause) => - Effect.logWarning("Audit queue send failed; writing directly", { cause }).pipe( - Effect.andThen(insertDirect(event)), - ), + // A Queues brown-out that stalls (rather than rejects) must not + // hang the mutation's response: 2s is far above a healthy send's + // latency yet bounds the worst case before the direct-write fallback. + Effect.timeout(AUDIT_QUEUE_SEND_TIMEOUT), + Effect.catchTag("TimeoutError", (error) => + Effect.fail(new AuditQueueSendError({ message: "Audit queue send timed out", cause: error })), + ), + Effect.catchTag("@maple/api/services/audit/AuditQueueSendError", (error) => + fallbackToDirect(event, error), ), ) @@ -180,9 +196,15 @@ export class AuditLogService extends Context.Service - Effect.logWarning("Audit log write failed", { action: input.action, cause }), + Effect.catch((error) => + Effect.logWarning("Audit log write failed", { action: input.action, cause: error }), + ), + Effect.catchDefect((defect) => + Effect.logWarning("Audit log write failed", { action: input.action, cause: defect }), ), ) }) diff --git a/apps/api/src/services/audit/audit-log-retention.ts b/apps/api/src/services/audit/audit-log-retention.ts index ec19b59ff..a1c176f05 100644 --- a/apps/api/src/services/audit/audit-log-retention.ts +++ b/apps/api/src/services/audit/audit-log-retention.ts @@ -1,8 +1,8 @@ import { auditLogEntries } from "@maple/db" -import { inArray, lt } from "drizzle-orm" +import { sql } from "drizzle-orm" import { Clock, Config, Effect } from "effect" import { Database } from "@/platform/DatabaseLive" -import { msToDate } from "@/platform/time" +import { msToSqlTimestamp } from "@/platform/time" /** * Retention for the org audit log (`audit_log_entries`). @@ -34,20 +34,21 @@ const retentionDaysConfig = Config.number("AUDIT_LOG_RETENTION_DAYS").pipe( export const runAuditLogRetention = Effect.gen(function* () { const retentionDays = yield* retentionDaysConfig const now = yield* Clock.currentTimeMillis - const cutoff = msToDate(now - retentionDays * DAY_MS) + // Raw-fragment param: bind an ISO string, not a Date — see msToSqlTimestamp. + const cutoff = msToSqlTimestamp(now - retentionDays * DAY_MS) const database = yield* Database const deleted = yield* database.execute(async (db) => { let total = 0 for (let batch = 0; batch < RETENTION_MAX_BATCHES; batch++) { - const staleIds = db - .select({ id: auditLogEntries.id }) - .from(auditLogEntries) - .where(lt(auditLogEntries.occurredAt, cutoff)) - .limit(RETENTION_BATCH_ROWS) + // ctid-addressed delete: one scan of the standalone occurred_at index + // finds the batch, and the DELETE fetches those exact tuples directly — + // no second lookup by a key the PK index (org_id, id) cannot serve. const rows = await db .delete(auditLogEntries) - .where(inArray(auditLogEntries.id, staleIds)) + .where( + sql`ctid IN (SELECT ctid FROM ${auditLogEntries} WHERE ${auditLogEntries.occurredAt} < ${cutoff}::timestamptz LIMIT ${RETENTION_BATCH_ROWS})`, + ) .returning({ id: auditLogEntries.id }) total += rows.length if (rows.length < RETENTION_BATCH_ROWS) break diff --git a/apps/api/src/services/auth/ApiAuthorizationLayer.ts b/apps/api/src/services/auth/ApiAuthorizationLayer.ts index 37f45cfde..27334c2be 100644 --- a/apps/api/src/services/auth/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationLayer.ts @@ -6,6 +6,7 @@ import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "@/services/auth/auth-span" import { CurrentAuditActor } from "@/services/auth/audit-actor" import { AuditLogService } from "@/services/audit/AuditLogService" +import { recordApiDenial } from "@/services/auth/audit-denial" import { Env } from "@/platform/Env" const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) @@ -47,18 +48,13 @@ export const ApiAuthorizationLayer = Layer.effect( const resolved = apiKeyResolved.value // Denied attempts are audited with the same attribution as // successes — a key probing a surface it is not valid for is - // exactly what the audit log exists to surface. + // exactly what the audit log exists to surface. This layer has + // no rate limiter, so coalescing is what bounds the volume. const recordDenied = (denialReason: string) => - audit.record({ + recordApiDenial(audit, request, { orgId: resolved.orgId, - actor: { - type: "api_key", - userId: resolved.userId, - apiKeyId: resolved.keyId, - }, - source: "api", - action: "api.request", - outcome: "denied", + userId: resolved.userId, + apiKeyId: resolved.keyId, denialReason, }) if (resolved.kind !== "standard") { diff --git a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts index 83117610d..b203095a1 100644 --- a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts @@ -17,6 +17,7 @@ import { OrgMembershipService } from "@/services/auth/OrgMembershipService" import { annotateAuthSpan } from "@/services/auth/auth-span" import { CurrentAuditActor } from "@/services/auth/audit-actor" import { AuditLogService } from "@/services/audit/AuditLogService" +import { recordApiDenial } from "@/services/auth/audit-denial" import { Env } from "@/platform/Env" import { API_V2_RATE_LIMIT_PERIOD_SECONDS, @@ -85,6 +86,17 @@ export const ApiAuthorizationV2Layer = Layer.effect( if (Option.isSome(apiKeyResolved)) { const resolved = apiKeyResolved.value + // A refused attempt is the highest-signal audit row there is — + // denials carry the same actor attribution as successes, tagged + // `outcome: "denied"`, coalesced so a looping client cannot + // amplify into unbounded rows. + const recordDenied = (denialReason: string) => + recordApiDenial(audit, request, { + orgId: resolved.orgId, + userId: resolved.userId, + apiKeyId: resolved.keyId, + denialReason, + }) // Deny-list, not an allow-list: `mcp` keys are minted through a // path that does not gate on organization admin, so they must // never reach the public API. `device` keys are admitted @@ -92,9 +104,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( // pinned roles below — is chosen by the server that minted // them, not by whatever is holding them. if (resolved.kind === "mcp") { - return yield* Effect.fail( - V2InvalidCredentials.make("This API key is only valid for the MCP server."), - ) + const message = "This API key is only valid for the MCP server." + yield* recordDenied(message) + return yield* Effect.fail(V2InvalidCredentials.make(message)) } // A device credential's authority is entirely its pinned @@ -103,9 +115,9 @@ export const ApiAuthorizationV2Layer = Layer.effect( // permissive default — it is a key whose defining property // is missing, so it is rejected rather than promoted. if (resolved.kind === "device" && resolved.roles === null) { - return yield* Effect.fail( - V2InvalidCredentials.make("This device credential is not valid."), - ) + const message = "This device credential is not valid." + yield* recordDenied(message) + return yield* Effect.fail(V2InvalidCredentials.make(message)) } // Attribute before the scope check so scope-rejected @@ -131,33 +143,6 @@ export const ApiAuthorizationV2Layer = Layer.effect( ) } - // A refused attempt is the highest-signal audit row there is — - // denials are recorded with the same actor attribution as - // successes, tagged `outcome: "denied"`. - const recordDenied = (denialReason: string) => - audit.record({ - orgId: resolved.orgId, - actor: { - type: "api_key", - userId: resolved.userId, - apiKeyId: resolved.keyId, - }, - source: "api", - action: "api.request", - outcome: "denied", - denialReason, - metadata: { method: request.method, path: requestPath(request.url) }, - ...(request.headers["cf-ray"] !== undefined - ? { requestId: request.headers["cf-ray"] } - : undefined), - ...(request.headers["cf-connecting-ip"] !== undefined - ? { originIp: request.headers["cf-connecting-ip"] } - : undefined), - ...(request.headers["cf-ipcountry"] !== undefined - ? { originCountry: request.headers["cf-ipcountry"] } - : undefined), - }) - const required = requiredScopeForRequest(request.method, requestPath(request.url)) if (required !== null && !scopeAllows(resolved.scopes, required)) { const message = `This API key does not have the "${required.family}:${required.access}" scope required for this request.` diff --git a/apps/api/src/services/auth/audit-denial.ts b/apps/api/src/services/auth/audit-denial.ts new file mode 100644 index 000000000..f8dd9352e --- /dev/null +++ b/apps/api/src/services/auth/audit-denial.ts @@ -0,0 +1,94 @@ +import { Clock, Effect } from "effect" +import type { HttpServerRequest } from "effect/unstable/http" +import type { ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import type { AuditLogServiceApi } from "@/services/audit/AuditLogService" + +/** Suppress duplicate denial rows for the same key/reason within this window. */ +export const AUDIT_DENIAL_COALESCE_WINDOW_MS = 60_000 + +/** Bound on distinct in-flight denial signatures kept per isolate. */ +const MAX_TRACKED_DENIALS = 10_000 + +/** + * Isolate-local coalescing cache: last-recorded time per denial signature. + * Tradeoff: Workers isolates multiply and recycle, so suppression is + * best-effort — each isolate still records the first denial it sees, which is + * the forensic signal; only the repeat volume is shed, with no network hop. + */ +const recentDenials = new Map() + +/** Test-only: clear the isolate-local coalescing state between cases. */ +export const resetAuditDenialCoalescing = (): void => { + recentDenials.clear() +} + +/** + * True when this signature has not been recorded within the window; marks it + * recorded. The timestamp is not refreshed on suppression, so a sustained loop + * still lands one row per window rather than going silent forever. + */ +const shouldRecordDenial = (signature: string, now: number): boolean => { + const last = recentDenials.get(signature) + if (last !== undefined && now - last < AUDIT_DENIAL_COALESCE_WINDOW_MS) return false + // Delete-then-set keeps insertion order ≈ recency, so the bound evicts the stalest signature. + recentDenials.delete(signature) + if (recentDenials.size >= MAX_TRACKED_DENIALS) { + const oldest = recentDenials.keys().next() + if (!oldest.done) recentDenials.delete(oldest.value) + } + recentDenials.set(signature, now) + return true +} + +export interface ApiDenialInput { + readonly orgId: OrgId + readonly userId: UserId + readonly apiKeyId: ApiKeyId + readonly denialReason: string +} + +const requestPath = (url: string): string => { + const queryStart = url.indexOf("?") + return queryStart === -1 ? url : url.slice(0, queryStart) +} + +/** + * Record a denied public-API request with full forensics (method+path plus the + * `cf-ray`/`cf-connecting-ip`/`cf-ipcountry` headers), coalescing duplicates: + * the same (org, key, method+path, reason) is written at most once per window + * so a client looping mis-scoped requests cannot amplify into unbounded queue + * messages, rows, and warn logs. Never fails — same contract as `record`. + */ +export const recordApiDenial = ( + audit: AuditLogServiceApi, + request: HttpServerRequest.HttpServerRequest, + input: ApiDenialInput, +): Effect.Effect => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const path = requestPath(request.url) + const signature = `${input.orgId}|${input.apiKeyId}|${request.method} ${path}|${input.denialReason}` + if (!shouldRecordDenial(signature, now)) return + yield* audit.record({ + orgId: input.orgId, + actor: { + type: "api_key", + userId: input.userId, + apiKeyId: input.apiKeyId, + }, + source: "api", + action: "api.request", + outcome: "denied", + denialReason: input.denialReason, + metadata: { method: request.method, path }, + ...(request.headers["cf-ray"] !== undefined + ? { requestId: request.headers["cf-ray"] } + : undefined), + ...(request.headers["cf-connecting-ip"] !== undefined + ? { originIp: request.headers["cf-connecting-ip"] } + : undefined), + ...(request.headers["cf-ipcountry"] !== undefined + ? { originCountry: request.headers["cf-ipcountry"] } + : undefined), + }) + }) diff --git a/apps/web/src/components/settings/settings-nav.tsx b/apps/web/src/components/settings/settings-nav.tsx index c6c8446c7..db51b7bf2 100644 --- a/apps/web/src/components/settings/settings-nav.tsx +++ b/apps/web/src/components/settings/settings-nav.tsx @@ -198,6 +198,9 @@ export function useVisibleSettingsSections() { ...section, items: section.items.filter((item) => { if (item.id === "data-platform") return canAccessDataPlatform + // `GET /v2/audit_log` is admin-only; hide the tab rather than let a + // member open it into a 403. + if (item.id === "audit-log") return isAdmin return true }), })) diff --git a/packages/db/drizzle/0050_audit_log_entries.sql b/packages/db/drizzle/0050_audit_log_entries.sql index fedc989f5..8247ac3a6 100644 --- a/packages/db/drizzle/0050_audit_log_entries.sql +++ b/packages/db/drizzle/0050_audit_log_entries.sql @@ -28,4 +28,11 @@ CREATE INDEX "audit_log_entries_org_occurred_idx" ON "audit_log_entries" USING b CREATE INDEX "audit_log_entries_org_actor_type_occurred_idx" ON "audit_log_entries" USING btree ("org_id","actor_type","occurred_at");--> statement-breakpoint CREATE INDEX "audit_log_entries_org_resource_idx" ON "audit_log_entries" USING btree ("org_id","resource_type","resource_id");--> statement-breakpoint CREATE INDEX "audit_log_entries_org_request_idx" ON "audit_log_entries" USING btree ("org_id","request_id");--> statement-breakpoint -CREATE INDEX "audit_log_entries_org_outcome_occurred_idx" ON "audit_log_entries" USING btree ("org_id","outcome","occurred_at"); \ No newline at end of file +CREATE INDEX "audit_log_entries_org_outcome_occurred_idx" ON "audit_log_entries" USING btree ("org_id","outcome","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_occurred_idx" ON "audit_log_entries" USING btree ("occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_user_occurred_idx" ON "audit_log_entries" USING btree ("org_id","user_id","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_api_key_occurred_idx" ON "audit_log_entries" USING btree ("org_id","api_key_id","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_actor_occurred_idx" ON "audit_log_entries" USING btree ("org_id","actor_id","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_affected_user_occurred_idx" ON "audit_log_entries" USING btree ("org_id","affected_user_id","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_org_action_occurred_idx" ON "audit_log_entries" USING btree ("org_id","action","occurred_at");--> statement-breakpoint +CREATE INDEX "audit_log_entries_changed_fields_gin_idx" ON "audit_log_entries" USING gin ("changed_fields"); \ No newline at end of file diff --git a/packages/db/drizzle/meta/0050_snapshot.json b/packages/db/drizzle/meta/0050_snapshot.json index 46076f621..caa784ce0 100644 --- a/packages/db/drizzle/meta/0050_snapshot.json +++ b/packages/db/drizzle/meta/0050_snapshot.json @@ -1,5 +1,5 @@ { - "id": "2b89a081-0fdd-4565-9366-89077aa29ec5", + "id": "5224691f-23a3-4172-b65b-be481ae93ad6", "prevId": "d60d7088-c27b-48cd-94b6-6d9fd59a02ff", "version": "7", "dialect": "postgresql", @@ -2080,6 +2080,171 @@ "concurrently": false, "method": "btree", "with": {} + }, + "audit_log_entries_occurred_idx": { + "name": "audit_log_entries_occurred_idx", + "columns": [ + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_user_occurred_idx": { + "name": "audit_log_entries_org_user_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_api_key_occurred_idx": { + "name": "audit_log_entries_org_api_key_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "api_key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_actor_occurred_idx": { + "name": "audit_log_entries_org_actor_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_affected_user_occurred_idx": { + "name": "audit_log_entries_org_affected_user_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "affected_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_org_action_occurred_idx": { + "name": "audit_log_entries_org_action_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_log_entries_changed_fields_gin_idx": { + "name": "audit_log_entries_changed_fields_gin_idx", + "columns": [ + { + "expression": "changed_fields", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} } }, "foreignKeys": {}, diff --git a/packages/db/src/schema/audit-log.ts b/packages/db/src/schema/audit-log.ts index c3990b67e..ca7e463ed 100644 --- a/packages/db/src/schema/audit-log.ts +++ b/packages/db/src/schema/audit-log.ts @@ -54,6 +54,25 @@ export const auditLogEntries = pgTable( table.outcome, table.occurredAt, ), + // Retention sweep scans `occurred_at < cutoff` across ALL orgs; every other + // index leads with org_id and cannot serve that predicate. + index("audit_log_entries_occurred_idx").on(table.occurredAt), + // "What did this credential do" — the primary read-endpoint filters. + index("audit_log_entries_org_user_occurred_idx").on(table.orgId, table.userId, table.occurredAt), + index("audit_log_entries_org_api_key_occurred_idx").on( + table.orgId, + table.apiKeyId, + table.occurredAt, + ), + index("audit_log_entries_org_actor_occurred_idx").on(table.orgId, table.actorId, table.occurredAt), + index("audit_log_entries_org_affected_user_occurred_idx").on( + table.orgId, + table.affectedUserId, + table.occurredAt, + ), + index("audit_log_entries_org_action_occurred_idx").on(table.orgId, table.action, table.occurredAt), + // drizzle `arrayContains` compiles to `@>`, which only GIN can serve on text[]. + index("audit_log_entries_changed_fields_gin_idx").using("gin", table.changedFields), ], ) diff --git a/packages/domain/src/http/audit-log.ts b/packages/domain/src/http/audit-log.ts index 19d572532..bafd703a8 100644 --- a/packages/domain/src/http/audit-log.ts +++ b/packages/domain/src/http/audit-log.ts @@ -41,6 +41,8 @@ export class AuditLogPersistenceError extends HttpTaggedError Date: Sat, 29 Aug 2026 17:34:08 +0200 Subject: [PATCH 04/19] chore(ios): regenerate OpenAPI spec for the audit log admin gate --- apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index f1155e639..c68b2076c 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -7336,7 +7336,7 @@ "name": "Attribute Mappings" }, { - "description": "The organization's append-only audit trail — every allowed or denied action performed through the dashboard, the public API, or MCP, attributed to the user, API key, or agent that performed it, with before/after diffs for updates.", + "description": "The organization's append-only audit trail — allowed and denied actions performed through the dashboard, the public API, and MCP, attributed to the user, API key, or agent that performed them, with before/after diffs for updates. Reading it requires organization-administrator access (or the `audit_log:read` scope for API keys).", "name": "Audit Log" }, { From dbf2f50f68ea2ecb38d0aad4b970477696064595 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 30 Aug 2026 00:55:20 +0200 Subject: [PATCH 05/19] refactor(audit): close the action namespace and derive the resource fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding an audited action meant restating what the action already implied: a free-string `resourceType` echoing the action's own prefix, an inline `encodePublicId(PublicIdPrefixes.x, id)`, and — for updates — a hand-assembled diff pipeline. Across 28 call sites the resource pair was mechanically derivable every time, and nothing checked it: the service's own test recorded `alert_rule.delete`, a verb that does not exist. `AuditResources` now declares each resource with its public-ID prefix and verbs, and `AuditAction` is the derived `${resource}.${verb}` union. `record`/`recordHttpAudit` take the internal ID and derive `resourceType` plus the public encoding themselves, so a typo fails the build, a `resourceId` on an org-singleton resource fails the build, and the prefix can no longer disagree with the resource. `error_issue` verbs come from `ErrorIssueEventType.literals` so a new issue event type cannot produce an undeclared action. `auditDiff({ fields, summarize, redact, writeOnly })` replaces the per-handler diff assembly; scrape-targets' update handler goes from ~40 lines of object surgery to one call. Keying `summarize`/`redact` by `fields` makes the old "remember to `satisfies`" rule structural. --- apps/api/src/mcp/tools/register-agent.ts | 4 +- .../src/routes/v2/alert-destinations.http.ts | 13 +-- apps/api/src/routes/v2/alert-rules.http.ts | 92 +++++++---------- apps/api/src/routes/v2/anomalies.http.ts | 6 +- apps/api/src/routes/v2/api-keys.http.ts | 11 +-- .../src/routes/v2/attribute-mappings.http.ts | 13 +-- apps/api/src/routes/v2/audit-changes.test.ts | 83 ++++++++++++++++ apps/api/src/routes/v2/audit-changes.ts | 66 ++++++++++++- apps/api/src/routes/v2/dashboards.http.ts | 67 +++++-------- apps/api/src/routes/v2/ingest-keys.http.ts | 2 - apps/api/src/routes/v2/scrape-targets.http.ts | 99 +++++-------------- .../services/audit/AuditLogService.test.ts | 13 ++- .../api/src/services/audit/AuditLogService.ts | 30 +++--- .../src/services/audit/audit-actions.test.ts | 39 ++++++++ apps/api/src/services/audit/audit-actions.ts | 95 ++++++++++++++++++ .../errors/ErrorIssueWorkflowService.ts | 4 +- 16 files changed, 405 insertions(+), 232 deletions(-) create mode 100644 apps/api/src/routes/v2/audit-changes.test.ts create mode 100644 apps/api/src/services/audit/audit-actions.test.ts create mode 100644 apps/api/src/services/audit/audit-actions.ts diff --git a/apps/api/src/mcp/tools/register-agent.ts b/apps/api/src/mcp/tools/register-agent.ts index 1e887c067..5e0006543 100644 --- a/apps/api/src/mcp/tools/register-agent.ts +++ b/apps/api/src/mcp/tools/register-agent.ts @@ -5,7 +5,6 @@ import { validationError, type McpToolRegistrar, } from "./types" -import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" import { Effect, Option, Schema } from "effect" import { createDualContent } from "@/mcp/lib/structured-output" import { CurrentMcpTenant } from "@/mcp/lib/query-warehouse" @@ -66,8 +65,7 @@ export function registerRegisterAgentTool(server: McpToolRegistrar) { actor: { type: "user", userId: tenant.userId }, source: "mcp", action: "agent.registered", - resourceType: "agent", - resourceId: encodePublicId(PublicIdPrefixes.actor, actor.id), + resourceId: actor.id, metadata: { name: actor.agentName ?? name }, }) diff --git a/apps/api/src/routes/v2/alert-destinations.http.ts b/apps/api/src/routes/v2/alert-destinations.http.ts index 4f586d98b..9f8325bba 100644 --- a/apps/api/src/routes/v2/alert-destinations.http.ts +++ b/apps/api/src/routes/v2/alert-destinations.http.ts @@ -18,7 +18,7 @@ import type { V2AlertDestinationUpdateParams, V2TelegramChatList, } from "@maple/domain/http/v2" -import { encodePublicId, MapleApiV2, paginateArray, PublicIdPrefixes } from "@maple/domain/http/v2" +import { MapleApiV2, paginateArray } from "@maple/domain/http/v2" import { Effect } from "effect" import { recordHttpAudit } from "@/services/audit/AuditLogService" import { AlertDestinationsService } from "@/services/alerts/AlertDestinationsService" @@ -301,8 +301,7 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale ) yield* recordHttpAudit("alert_destination.created", { - resourceType: "alert_destination", - resourceId: encodePublicId(PublicIdPrefixes.alertDestination, created.id), + resourceId: created.id, metadata: { name: created.name, type: created.type }, }) @@ -325,9 +324,8 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale const changes = buildDestinationChanges(request, current, updated) yield* recordHttpAudit("alert_destination.updated", { - resourceType: "alert_destination", - resourceId: encodePublicId(PublicIdPrefixes.alertDestination, updated.id), - ...(changes !== undefined ? { changes } : undefined), + resourceId: updated.id, + changes, metadata: { name: updated.name, type: updated.type }, }) @@ -343,8 +341,7 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale params.id, ) yield* recordHttpAudit("alert_destination.deleted", { - resourceType: "alert_destination", - resourceId: encodePublicId(PublicIdPrefixes.alertDestination, deleted.id), + resourceId: deleted.id, }) return { diff --git a/apps/api/src/routes/v2/alert-rules.http.ts b/apps/api/src/routes/v2/alert-rules.http.ts index fe1f5122d..abfb40c5b 100644 --- a/apps/api/src/routes/v2/alert-rules.http.ts +++ b/apps/api/src/routes/v2/alert-rules.http.ts @@ -16,18 +16,10 @@ import type { V2AlertRulePreviewResult, V2AlertRuleUpdateParams, } from "@maple/domain/http/v2" -import { - encodePublicId, - MapleApiV2, - paginateArray, - PublicIdPrefixes, - scopeAllows, - timestamp, - V2ParameterInvalid, -} from "@maple/domain/http/v2" +import { MapleApiV2, paginateArray, scopeAllows, timestamp, V2ParameterInvalid } from "@maple/domain/http/v2" import { AlertForbiddenError } from "@maple/domain/http" import { Effect, Encoding, Result, Schema } from "effect" -import { compactAuditChanges, diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { auditDiff } from "@/routes/v2/audit-changes" import { recordHttpAudit } from "@/services/audit/AuditLogService" import { AlertsService } from "@/services/alerts/AlertsService" import { AlertReadModelsService } from "@/services/alerts/AlertReadModelsService" @@ -105,33 +97,36 @@ const toV2Rule = (doc: AlertRuleDocument): V2AlertRule => ({ }) /** Update-payload fields diffable through the wire shape (drafts get summarized). */ -const ruleAuditKeys: ReadonlyArray = [ - "name", - "notes", - "notification_template", - "enabled", - "severity", - "service_names", - "exclude_service_names", - "environments", - "tags", - "group_by", - "signal_type", - "comparator", - "threshold", - "threshold_upper", - "window_minutes", - "minimum_sample_count", - "consecutive_breaches_required", - "consecutive_healthy_required", - "renotify_interval_minutes", - "apdex_threshold_ms", - "query_builder_draft", - "raw_query_sql", - "raw_query_reducer", - "destination_ids", -] - +const ruleAuditDiff = auditDiff({ + fields: [ + "name", + "notes", + "notification_template", + "enabled", + "severity", + "service_names", + "exclude_service_names", + "environments", + "tags", + "group_by", + "signal_type", + "comparator", + "threshold", + "threshold_upper", + "window_minutes", + "minimum_sample_count", + "consecutive_breaches_required", + "consecutive_healthy_required", + "renotify_interval_minutes", + "apdex_threshold_ms", + "query_builder_draft", + "raw_query_sql", + "raw_query_reducer", + "destination_ids", + ], + // Query drafts and raw SQL are config blobs — audit that they changed, not their bodies. + summarize: { query_builder_draft: "", raw_query_sql: "" }, +}) const toV2RuleMutationResponse = (doc: AlertRuleDocument): V2AlertRuleMutationResponse => ({ ...toV2Rule(doc), @@ -385,8 +380,7 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules ) yield* recordHttpAudit("alert_rule.created", { - resourceType: "alert_rule", - resourceId: encodePublicId(PublicIdPrefixes.alertRule, created.id), + resourceId: created.id, metadata: { name: created.name }, }) @@ -406,20 +400,9 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules request, ) - const changes = compactAuditChanges( - diffAuditChanges( - pickPresentFields(ruleAuditKeys, payload, toV2Rule(current)), - pickPresentFields(ruleAuditKeys, payload, toV2Rule(updated)), - ), - // Query drafts and raw SQL are config blobs — audit that they changed, not their bodies. - { query_builder_draft: "", raw_query_sql: "" } satisfies Partial< - Record<(typeof ruleAuditKeys)[number], string> - >, - ) yield* recordHttpAudit("alert_rule.updated", { - resourceType: "alert_rule", - resourceId: encodePublicId(PublicIdPrefixes.alertRule, updated.id), - ...(changes !== undefined ? { changes } : undefined), + resourceId: updated.id, + changes: ruleAuditDiff(payload, toV2Rule(current), toV2Rule(updated)), metadata: { name: updated.name }, }) @@ -430,10 +413,7 @@ export const HttpV2AlertRulesLive = HttpApiBuilder.group(MapleApiV2, "alertRules Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* rules.deleteRule(tenant.orgId, tenant.roles, params.id) - yield* recordHttpAudit("alert_rule.deleted", { - resourceType: "alert_rule", - resourceId: encodePublicId(PublicIdPrefixes.alertRule, deleted.id), - }) + yield* recordHttpAudit("alert_rule.deleted", { resourceId: deleted.id }) return { id: deleted.id, diff --git a/apps/api/src/routes/v2/anomalies.http.ts b/apps/api/src/routes/v2/anomalies.http.ts index 1cfc5dad2..62049a999 100644 --- a/apps/api/src/routes/v2/anomalies.http.ts +++ b/apps/api/src/routes/v2/anomalies.http.ts @@ -12,7 +12,7 @@ import { AnomalyForbiddenError, CurrentTenant, } from "@maple/domain/http" -import { encodePublicId, MapleApiV2, paginateOffsetQuery, PublicIdPrefixes, timestamp } from "@maple/domain/http/v2" +import { MapleApiV2, paginateOffsetQuery, timestamp } from "@maple/domain/http/v2" import type { V2AnomalyIncident, V2AnomalyIncidentTimeseries, V2AnomalySettings } from "@maple/domain/http/v2" import { Effect } from "effect" import { recordHttpAudit } from "@/services/audit/AuditLogService" @@ -188,8 +188,7 @@ export const HttpV2AnomaliesLive = HttpApiBuilder.group(MapleApiV2, "anomalies", const tenant = yield* CurrentTenant.Context const incident = yield* anomalies.resolveIncidentManually(tenant.orgId, params.id) yield* recordHttpAudit("anomaly_incident.resolved", { - resourceType: "anomaly_incident", - resourceId: encodePublicId(PublicIdPrefixes.anomalyIncident, incident.id), + resourceId: incident.id, metadata: { signal_type: incident.signalType, service_name: incident.serviceName, @@ -254,7 +253,6 @@ export const HttpV2AnomaliesLive = HttpApiBuilder.group(MapleApiV2, "anomalies", ) yield* recordHttpAudit("anomaly_settings.updated", { - resourceType: "anomaly_settings", metadata: { enabled: settings.enabled, sensitivity: settings.sensitivity }, }) diff --git a/apps/api/src/routes/v2/api-keys.http.ts b/apps/api/src/routes/v2/api-keys.http.ts index 152742d7d..0e737a3e5 100644 --- a/apps/api/src/routes/v2/api-keys.http.ts +++ b/apps/api/src/routes/v2/api-keys.http.ts @@ -2,12 +2,10 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import type { ApiKeyCreatedResponse, ApiKeyResponse } from "@maple/domain/http" import { CurrentTenant } from "@maple/domain/http" import { - encodePublicId, MapleApiV2, isoTimestamp, isoTimestampOrNull, paginateArray, - PublicIdPrefixes, V2InsufficientPermissions, } from "@maple/domain/http/v2" import type { V2ApiKey, V2ApiKeyMutationResponse, V2ApiKeyWithSecret } from "@maple/domain/http/v2" @@ -110,8 +108,7 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha : undefined), }) yield* recordHttpAudit("api_key.created", { - resourceType: "api_key", - resourceId: encodePublicId(PublicIdPrefixes.apiKey, created.id), + resourceId: created.id, metadata: { name: created.name, kind: created.kind, scopes: created.scopes }, }) return toV2ApiKeyWithSecret(created) @@ -126,8 +123,7 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha createdByEmail, }) yield* recordHttpAudit("api_key.rolled", { - resourceType: "api_key", - resourceId: encodePublicId(PublicIdPrefixes.apiKey, rolled.id), + resourceId: rolled.id, metadata: { name: rolled.name, scopes: rolled.scopes }, }) return toV2ApiKeyWithSecret(rolled) @@ -146,8 +142,7 @@ export const HttpV2ApiKeysLive = HttpApiBuilder.group(MapleApiV2, "apiKeys", (ha } const revoked = yield* apiKeysService.revoke(tenant.orgId, params.id) yield* recordHttpAudit("api_key.revoked", { - resourceType: "api_key", - resourceId: encodePublicId(PublicIdPrefixes.apiKey, revoked.id), + resourceId: revoked.id, metadata: { name: revoked.name }, }) return toV2ApiKeyMutationResponse(revoked) diff --git a/apps/api/src/routes/v2/attribute-mappings.http.ts b/apps/api/src/routes/v2/attribute-mappings.http.ts index 4557de176..948e70b90 100644 --- a/apps/api/src/routes/v2/attribute-mappings.http.ts +++ b/apps/api/src/routes/v2/attribute-mappings.http.ts @@ -6,7 +6,7 @@ import { IngestAttributeMappingNotFoundError, UpdateIngestAttributeMappingRequest, } from "@maple/domain/http" -import { encodePublicId, MapleApiV2, paginateArray, PublicIdPrefixes } from "@maple/domain/http/v2" +import { MapleApiV2, paginateArray } from "@maple/domain/http/v2" import type { V2AttributeMapping } from "@maple/domain/http/v2" import { Array as Arr, Effect, Option } from "effect" import { diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" @@ -88,8 +88,7 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att ) yield* recordHttpAudit("attribute_mapping.created", { - resourceType: "attribute_mapping", - resourceId: encodePublicId(PublicIdPrefixes.attributeMapping, created.id), + resourceId: created.id, metadata: { name: created.name }, }) @@ -128,9 +127,8 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att pickPresentFields(mappingAuditKeys, payload, toV2AttributeMapping(updated)), ) yield* recordHttpAudit("attribute_mapping.updated", { - resourceType: "attribute_mapping", - resourceId: encodePublicId(PublicIdPrefixes.attributeMapping, updated.id), - ...(changes !== undefined ? { changes } : undefined), + resourceId: updated.id, + changes, metadata: { name: updated.name }, }) @@ -142,8 +140,7 @@ export const HttpV2AttributeMappingsLive = HttpApiBuilder.group(MapleApiV2, "att const tenant = yield* CurrentTenant.Context const deleted = yield* service.delete(tenant.orgId, params.id) yield* recordHttpAudit("attribute_mapping.deleted", { - resourceType: "attribute_mapping", - resourceId: encodePublicId(PublicIdPrefixes.attributeMapping, deleted.id), + resourceId: deleted.id, }) return { id: deleted.id, object: "attribute_mapping" as const, deleted: true as const } diff --git a/apps/api/src/routes/v2/audit-changes.test.ts b/apps/api/src/routes/v2/audit-changes.test.ts new file mode 100644 index 000000000..a7f31fd58 --- /dev/null +++ b/apps/api/src/routes/v2/audit-changes.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest" +import { auditDiff, redactAuditUrl } from "./audit-changes" + +const targetDiff = auditDiff({ + fields: ["name", "url", "enabled", "labels_json"], + summarize: { labels_json: "" }, + redact: { url: redactAuditUrl }, + writeOnly: ["auth_credentials"], +}) + +describe("auditDiff", () => { + it("diffs only the fields the payload carried", () => { + const changes = targetDiff( + { name: "renamed" }, + { name: "before", url: "https://a.test/x", enabled: true, labels_json: "{}" }, + { name: "renamed", url: "https://b.test/y", enabled: false, labels_json: "{}" }, + ) + // `url` and `enabled` moved, but the request did not ask for them. + expect(changes).toEqual({ + fields: ["name"], + before: { name: "before" }, + after: { name: "renamed" }, + }) + }) + + it("returns undefined when a touched field is unchanged", () => { + expect( + targetDiff( + { name: "same" }, + { name: "same", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "same", url: "https://a.test", enabled: true, labels_json: "{}" }, + ), + ).toBeUndefined() + }) + + it("redacts credentials out of a changed URL", () => { + const changes = targetDiff( + { url: "https://user:secret@b.test/m?token=live" }, + { name: "n", url: "https://a.test/m", enabled: true, labels_json: "{}" }, + { name: "n", url: "https://user:secret@b.test/m?token=live", enabled: true, labels_json: "{}" }, + ) + expect(changes?.after["url"]).toBe("https://b.test/m") + expect(JSON.stringify(changes)).not.toContain("secret") + expect(JSON.stringify(changes)).not.toContain("token=live") + }) + + it("summarizes config blobs instead of recording their bodies", () => { + const changes = targetDiff( + { labels_json: '{"team":"infra"}' }, + { name: "n", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "n", url: "https://a.test", enabled: true, labels_json: '{"team":"infra"}' }, + ) + expect(changes).toEqual({ + fields: ["labels_json"], + before: { labels_json: "" }, + after: { labels_json: "" }, + }) + }) + + it("records a write-only field as rotated whenever the payload carries it", () => { + const changes = targetDiff( + { auth_credentials: "hunter2" }, + { name: "n", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "n", url: "https://a.test", enabled: true, labels_json: "{}" }, + ) + expect(changes).toEqual({ + fields: ["auth_credentials"], + before: { auth_credentials: "" }, + after: { auth_credentials: "" }, + }) + expect(JSON.stringify(changes)).not.toContain("hunter2") + }) + + it("merges a rotated credential into an observable diff", () => { + const changes = targetDiff( + { name: "renamed", auth_credentials: "hunter2" }, + { name: "before", url: "https://a.test", enabled: true, labels_json: "{}" }, + { name: "renamed", url: "https://a.test", enabled: true, labels_json: "{}" }, + ) + expect(changes?.fields).toEqual(["name", "auth_credentials"]) + expect(changes?.after).toEqual({ name: "renamed", auth_credentials: "" }) + }) +}) diff --git a/apps/api/src/routes/v2/audit-changes.ts b/apps/api/src/routes/v2/audit-changes.ts index 40853fd03..4986d06dc 100644 --- a/apps/api/src/routes/v2/audit-changes.ts +++ b/apps/api/src/routes/v2/audit-changes.ts @@ -64,9 +64,9 @@ export const pickPresentFields = ( */ export const compactAuditChanges = ( changes: AuditChanges | undefined, - // Call sites `satisfies Partial>` - // so a wire-key rename cannot silently disable a redaction placeholder. - placeholders: Record, + // Keyed by the resource's declared field names (see `auditDiff`) so a wire-key + // rename cannot silently disable a placeholder. + placeholders: Record, ): AuditChanges | undefined => { if (changes === undefined) return undefined const before = { ...changes.before } @@ -89,3 +89,63 @@ export const redactAuditUrl = (raw: string): string => { const url = new URL(raw) return `${url.protocol}//${url.host}${url.pathname}` } + +/** + * Build the `changes` diff for one resource's update handler. + * + * The spec is declared once next to the resource's wire shape and applied per + * request: `fields` are diffed through the wire view, `summarize` replaces a + * config blob's value with a static placeholder, `redact` rewrites a value + * (scrape URLs carry tokens), and `writeOnly` records credentials the response + * never echoes as having rotated. `summarize` and `redact` are keyed by + * `fields`, so a renamed wire key is a type error rather than a silently + * disabled redaction. + * + * Returns undefined when nothing observable changed, so the caller passes the + * result straight through as `changes`. + */ +export const auditDiff = (spec: { + readonly fields: ReadonlyArray + readonly summarize?: Partial> + readonly redact?: Partial string>> + readonly writeOnly?: ReadonlyArray +}) => { + const redactors: Record string) | undefined> = spec.redact ?? {} + + const redactChanges = (changes: AuditChanges): AuditChanges => { + const apply = (values: Record): Record => { + const out = { ...values } + for (const field of changes.fields) { + const redact = redactors[field] + const value = out[field] + if (redact !== undefined && typeof value === "string") out[field] = redact(value) + } + return out + } + return { fields: changes.fields, before: apply(changes.before), after: apply(changes.after) } + } + + return ( + payload: { readonly [P in Field]?: unknown }, + before: { readonly [P in Field]: unknown }, + after: { readonly [P in Field]: unknown }, + ): AuditChanges | undefined => { + const diffed = diffAuditChanges( + pickPresentFields(spec.fields, payload, before), + pickPresentFields(spec.fields, payload, after), + ) + const compacted = diffed === undefined ? undefined : compactAuditChanges(diffed, spec.summarize ?? {}) + const observable = compacted === undefined ? undefined : redactChanges(compacted) + // Write-only fields never appear in a response, so their rotation can only + // be inferred from the request carrying them. + const present: Record = payload + const rotated = (spec.writeOnly ?? []).filter((field) => present[field] !== undefined) + if (rotated.length === 0) return observable + const placeholders = Object.fromEntries(rotated.map((field) => [field, ""])) + return { + fields: [...(observable?.fields ?? []), ...rotated], + before: { ...observable?.before, ...placeholders }, + after: { ...observable?.after, ...placeholders }, + } + } +} diff --git a/apps/api/src/routes/v2/dashboards.http.ts b/apps/api/src/routes/v2/dashboards.http.ts index 6ba9b87c0..1a9420477 100644 --- a/apps/api/src/routes/v2/dashboards.http.ts +++ b/apps/api/src/routes/v2/dashboards.http.ts @@ -32,7 +32,7 @@ import type { DashboardId } from "@maple/domain/primitives" import { Clock, Effect, Option, Schema } from "effect" import { getTemplateById, listTemplateMetadata } from "@/dashboard-templates" import type { TemplateParameterValues } from "@/dashboard-templates" -import { compactAuditChanges, diffAuditChanges, pickPresentFields } from "@/routes/v2/audit-changes" +import { auditDiff } from "@/routes/v2/audit-changes" import { recordHttpAudit } from "@/services/audit/AuditLogService" import { DashboardPersistenceService } from "@/services/dashboards/DashboardPersistenceService" import { SharedDashboardService } from "@/services/dashboards/SharedDashboardService" @@ -179,17 +179,20 @@ const applyUpdate = ( } /** Update-payload fields diffable through the wire shape; layout blobs get summarized. */ -const dashboardAuditKeys: ReadonlyArray = [ - "name", - "description", - "tags", - "timeRange", - "widgets", - "sections", - "variables", - "refreshIntervalSeconds", -] - +const dashboardAuditDiff = auditDiff({ + fields: [ + "name", + "description", + "tags", + "timeRange", + "widgets", + "sections", + "variables", + "refreshIntervalSeconds", + ], + // Layout arrays are config blobs — audit that they changed, not their bodies. + summarize: { widgets: "", sections: "", variables: "" }, +}) const encodeVersionCursor = (versionNumber: number): string => `ver_${versionNumber.toString(36)}` @@ -288,8 +291,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards mode: created.mode, }) yield* recordHttpAudit("dashboard_share.created", { - resourceType: "dashboard_share", - resourceId: encodePublicId(PublicIdPrefixes.dashboardShare, created.id), + resourceId: created.id, metadata: { mode: created.mode, dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, context.scope.dashboardId), @@ -314,8 +316,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards yield* logShare("dashboard share rotated", context, { "maple.share.id": rotated.id }) // Security event: rotation invalidates the previous public share token. yield* recordHttpAudit("dashboard_share.rotated", { - resourceType: "dashboard_share", - resourceId: encodePublicId(PublicIdPrefixes.dashboardShare, rotated.id), + resourceId: rotated.id, metadata: { dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, dashboardId), ...(widgetId === null ? undefined : { widget_id: widgetId }), @@ -337,7 +338,6 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards yield* logShare("dashboard share revoked", context, { hadLiveShare: tombstone.revoked }) if (tombstone.revoked) { yield* recordHttpAudit("dashboard_share.deleted", { - resourceType: "dashboard_share", metadata: { dashboard_id: encodePublicId(PublicIdPrefixes.dashboard, dashboardId), ...(widgetId === null ? undefined : { widget_id: widgetId }), @@ -383,8 +383,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards toPortable(payload), ) yield* recordHttpAudit("dashboard.created", { - resourceType: "dashboard", - resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + resourceId: dashboard.id, metadata: { name: dashboard.name }, }) @@ -411,22 +410,10 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards const changes = previous === undefined ? undefined - : compactAuditChanges( - diffAuditChanges( - pickPresentFields(dashboardAuditKeys, payload, toV2Dashboard(previous)), - pickPresentFields(dashboardAuditKeys, payload, toV2Dashboard(dashboard)), - ), - // Layout arrays are config blobs — audit that they changed, not their bodies. - { - widgets: "", - sections: "", - variables: "", - } satisfies Partial>, - ) + : dashboardAuditDiff(payload, toV2Dashboard(previous), toV2Dashboard(dashboard)) yield* recordHttpAudit("dashboard.updated", { - resourceType: "dashboard", - resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), - ...(changes !== undefined ? { changes } : undefined), + resourceId: dashboard.id, + changes, metadata: { name: dashboard.name }, }) @@ -438,8 +425,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards const tenant = yield* CurrentTenant.Context const deleted = yield* persistence.delete(tenant.orgId, params.id) yield* recordHttpAudit("dashboard.deleted", { - resourceType: "dashboard", - resourceId: encodePublicId(PublicIdPrefixes.dashboard, deleted.id), + resourceId: deleted.id, }) return { @@ -460,8 +446,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards converted.dashboard, ) yield* recordHttpAudit("dashboard.created", { - resourceType: "dashboard", - resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + resourceId: dashboard.id, metadata: { name: dashboard.name, source: "perses_import" }, }) @@ -523,8 +508,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards params.version_id, ) yield* recordHttpAudit("dashboard.version_restored", { - resourceType: "dashboard", - resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + resourceId: dashboard.id, metadata: { name: dashboard.name, version_id: encodePublicId( @@ -626,8 +610,7 @@ export const HttpV2DashboardsLive = HttpApiBuilder.group(MapleApiV2, "dashboards const tenant = yield* CurrentTenant.Context const dashboard = yield* persistence.create(tenant.orgId, tenant.userId, portable) yield* recordHttpAudit("dashboard.created", { - resourceType: "dashboard", - resourceId: encodePublicId(PublicIdPrefixes.dashboard, dashboard.id), + resourceId: dashboard.id, metadata: { name: dashboard.name, source: "template", diff --git a/apps/api/src/routes/v2/ingest-keys.http.ts b/apps/api/src/routes/v2/ingest-keys.http.ts index b337fd79c..28fa08c31 100644 --- a/apps/api/src/routes/v2/ingest-keys.http.ts +++ b/apps/api/src/routes/v2/ingest-keys.http.ts @@ -39,7 +39,6 @@ export const HttpV2IngestKeysLive = HttpApiBuilder.group(MapleApiV2, "ingestKeys yield* requireAdmin(tenant.roles, adminOnly("roll")) const keys = yield* ingestKeys.rerollPublic(tenant.orgId, tenant.userId) yield* recordHttpAudit("ingest_key.rolled", { - resourceType: "ingest_key", metadata: { key_type: "public" }, }) @@ -52,7 +51,6 @@ export const HttpV2IngestKeysLive = HttpApiBuilder.group(MapleApiV2, "ingestKeys yield* requireAdmin(tenant.roles, adminOnly("roll")) const keys = yield* ingestKeys.rerollPrivate(tenant.orgId, tenant.userId) yield* recordHttpAudit("ingest_key.rolled", { - resourceType: "ingest_key", metadata: { key_type: "private" }, }) diff --git a/apps/api/src/routes/v2/scrape-targets.http.ts b/apps/api/src/routes/v2/scrape-targets.http.ts index 71fee0fd6..755d0ba12 100644 --- a/apps/api/src/routes/v2/scrape-targets.http.ts +++ b/apps/api/src/routes/v2/scrape-targets.http.ts @@ -1,17 +1,10 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import type { ScrapeTargetResponse } from "@maple/domain/http" import { CreateScrapeTargetRequest, CurrentTenant, UpdateScrapeTargetRequest } from "@maple/domain/http" -import { - encodePublicId, - MapleApiV2, - paginateArray, - paginateOffsetQuery, - PublicIdPrefixes, - timestamp, -} from "@maple/domain/http/v2" +import { MapleApiV2, paginateArray, paginateOffsetQuery, timestamp } from "@maple/domain/http/v2" import type { V2ScrapeTarget, V2ScrapeTargetCheck } from "@maple/domain/http/v2" import { Effect } from "effect" -import { diffAuditChanges, pickPresentFields, redactAuditUrl } from "@/routes/v2/audit-changes" +import { auditDiff, redactAuditUrl } from "@/routes/v2/audit-changes" import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ScrapeTargetsService } from "@/services/integrations/ScrapeTargetsService" @@ -38,29 +31,25 @@ const toV2ScrapeTarget = (target: ScrapeTargetResponse): V2ScrapeTarget => ({ }) /** Update-payload fields diffable through the wire shape; credentials never appear. */ -const targetAuditKeys: ReadonlyArray< - | "name" - | "url" - | "organization" - | "include_branches" - | "exclude_branches" - | "scrape_interval_seconds" - | "labels_json" - | "auth_type" - | "service_name" - | "enabled" -> = [ - "name", - "url", - "organization", - "include_branches", - "exclude_branches", - "scrape_interval_seconds", - "labels_json", - "auth_type", - "service_name", - "enabled", -] +const targetAuditDiff = auditDiff({ + fields: [ + "name", + "url", + "organization", + "include_branches", + "exclude_branches", + "scrape_interval_seconds", + "labels_json", + "auth_type", + "service_name", + "enabled", + ], + // Scrape URLs may carry tokens in userinfo/query — audit only scheme/host/path. + // Identical redacted values still mean the URL changed within the stripped part. + redact: { url: redactAuditUrl }, + // Credentials are write-only: audit that they rotated, never their value. + writeOnly: ["auth_credentials"], +}) export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeTargets", (handlers) => Effect.gen(function* () { @@ -131,8 +120,7 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT ) yield* recordHttpAudit("scrape_target.created", { - resourceType: "scrape_target", - resourceId: encodePublicId(PublicIdPrefixes.scrapeTarget, created.id), + resourceId: created.id, metadata: { name: created.name }, }) @@ -187,43 +175,9 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT // Read-then-write with no CAS: a concurrent update can make `before` // reflect a state this update never saw. Accepted for audit purposes. - const diffed = diffAuditChanges( - pickPresentFields(targetAuditKeys, payload, toV2ScrapeTarget(current)), - pickPresentFields(targetAuditKeys, payload, toV2ScrapeTarget(updated)), - ) - // Scrape URLs may carry tokens in userinfo/query — audit only scheme/host/path. - // Identical redacted values still mean the URL changed within the stripped part. - const observable = - diffed === undefined || !diffed.fields.includes("url") - ? diffed - : { - fields: diffed.fields, - before: { - ...diffed.before, - ...(typeof diffed.before["url"] === "string" - ? { url: redactAuditUrl(diffed.before["url"]) } - : undefined), - }, - after: { - ...diffed.after, - ...(typeof diffed.after["url"] === "string" - ? { url: redactAuditUrl(diffed.after["url"]) } - : undefined), - }, - } - // Credentials are write-only: audit that they rotated, never their value. - const changes = - payload.auth_credentials !== undefined - ? { - fields: [...(observable?.fields ?? []), "auth_credentials"], - before: { ...observable?.before, auth_credentials: "" }, - after: { ...observable?.after, auth_credentials: "" }, - } - : observable yield* recordHttpAudit("scrape_target.updated", { - resourceType: "scrape_target", - resourceId: encodePublicId(PublicIdPrefixes.scrapeTarget, updated.id), - ...(changes !== undefined ? { changes } : undefined), + resourceId: updated.id, + changes: targetAuditDiff(payload, toV2ScrapeTarget(current), toV2ScrapeTarget(updated)), metadata: { name: updated.name }, }) @@ -234,10 +188,7 @@ export const HttpV2ScrapeTargetsLive = HttpApiBuilder.group(MapleApiV2, "scrapeT Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const deleted = yield* service.delete(tenant.orgId, params.id) - yield* recordHttpAudit("scrape_target.deleted", { - resourceType: "scrape_target", - resourceId: encodePublicId(PublicIdPrefixes.scrapeTarget, deleted.id), - }) + yield* recordHttpAudit("scrape_target.deleted", { resourceId: deleted.id }) return { id: deleted.id, object: "scrape_target" as const, deleted: true as const } }), diff --git a/apps/api/src/services/audit/AuditLogService.test.ts b/apps/api/src/services/audit/AuditLogService.test.ts index 267d965a4..543d16948 100644 --- a/apps/api/src/services/audit/AuditLogService.test.ts +++ b/apps/api/src/services/audit/AuditLogService.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from "@effect/vitest" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" import { OrgId, UserId } from "@maple/domain/primitives" import { Effect, Layer, Schema } from "effect" import { TestClock } from "effect/testing" @@ -15,6 +16,8 @@ const createdDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(createdDbs)) +const DASHBOARD_ID = "3f1b7c02-9a44-4d1e-8b2f-0c5d6e7a8b91" + const makeLayer = () => AuditLogService.layer.pipe(Layer.provide(createTestDb(createdDbs).layer)) /** Three entries with distinct timestamps: user, then api_key, then agent. */ @@ -25,8 +28,8 @@ const seedThree = Effect.gen(function* () { actor: { type: "user", userId: USER }, source: "dashboard", action: "dashboard.created", - resourceType: "dashboard", - resourceId: "dash_first", + // Internal ID in, public `dash_…` ID out — the service owns the encoding. + resourceId: DASHBOARD_ID, metadata: { name: "First" }, }) yield* TestClock.adjust("1 second") @@ -63,7 +66,7 @@ describe("AuditLogService", () => { expect(oldest.userId).toBe(USER) expect(oldest.source).toBe("dashboard") expect(oldest.resourceType).toBe("dashboard") - expect(oldest.resourceId).toBe("dash_first") + expect(oldest.resourceId).toBe(encodePublicId(PublicIdPrefixes.dashboard, DASHBOARD_ID)) expect(oldest.metadataJson).toEqual({ name: "First" }) const newest = rows[0]! @@ -109,13 +112,13 @@ describe("AuditLogService", () => { orgId: ORG, actor: { type: "user", userId: USER }, source: "dashboard", - action: "alert_rule.delete", + action: "alert_rule.deleted", outcome: "denied", denialReason: "missing role: admin", }) const denied = yield* audit.list(ORG, { outcome: "denied", limit: 10, offset: 0 }) - expect(denied.map((row) => row.action)).toEqual(["alert_rule.delete"]) + expect(denied.map((row) => row.action)).toEqual(["alert_rule.deleted"]) expect(denied[0]!.outcome).toBe("denied") expect(denied[0]!.denialReason).toBe("missing role: admin") diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index fcf2cf5ce..8ebc4110d 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -12,6 +12,7 @@ import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" import { Database, type DatabaseError } from "@/platform/DatabaseLive" import { msToDate } from "@/platform/time" import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { type AuditAction, auditResourceFields, type AuditResourceIdOption } from "./audit-actions" import { AuditLogEvent, auditEventToInsert, encodeAuditLogEventSync } from "./audit-event" const decodeAuditLogEntryIdSync = Schema.decodeUnknownSync(AuditLogEntryIdSchema) @@ -46,24 +47,22 @@ export interface AuditActorRef { readonly label?: string } -export interface AuditLogRecordInput { +export type AuditLogRecordInput = { readonly orgId: OrgId readonly actor: AuditActorRef readonly source: AuditLogSource - /** `.`, e.g. `alert_rule.created`. */ - readonly action: string + /** Declared in `AuditResources`; the row's `resource_type` is derived from it. */ + readonly action: A /** Defaults to `"allowed"`; denied attempts pass `"denied"` + `denialReason`. */ readonly outcome?: AuditOutcome readonly denialReason?: string readonly affectedUserId?: UserId - readonly resourceType?: string - readonly resourceId?: string - readonly changes?: AuditChanges + readonly changes?: AuditChanges | undefined readonly metadata?: Record readonly requestId?: string readonly originIp?: string readonly originCountry?: string -} +} & AuditResourceIdOption export interface AuditLogListFilters { readonly actorType?: AuditActorType @@ -94,7 +93,7 @@ export interface AuditLogServiceApi { * Never fails: a mutation that succeeded must not 500 because its audit * write did not — terminal failures are logged and swallowed. */ - readonly record: (input: AuditLogRecordInput) => Effect.Effect + readonly record: (input: AuditLogRecordInput) => Effect.Effect readonly list: ( orgId: OrgId, filters: AuditLogListFilters, @@ -158,6 +157,7 @@ export class AuditLogService extends Context.Service( + action: A, opts?: { - readonly resourceType?: string - readonly resourceId?: string - readonly changes?: AuditChanges + readonly changes?: AuditChanges | undefined readonly affectedUserId?: UserId readonly metadata?: Record - }, + } & AuditResourceIdOption, ) => Effect.gen(function* () { const audit = yield* AuditLogService diff --git a/apps/api/src/services/audit/audit-actions.test.ts b/apps/api/src/services/audit/audit-actions.test.ts new file mode 100644 index 000000000..b72044316 --- /dev/null +++ b/apps/api/src/services/audit/audit-actions.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest" +import { decodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" +import { ErrorIssueEventType } from "@maple/domain/http" +import { AuditResources, auditResourceFields } from "./audit-actions" + +describe("AuditResources", () => { + it("names every resource in the `.` snake_case shape the rows store", () => { + for (const [resource, { verbs }] of Object.entries(AuditResources)) { + expect(resource).toMatch(/^[a-z][a-z0-9_]*$/) + for (const verb of verbs) expect(verb).toMatch(/^[a-z][a-z0-9_]*$/) + } + }) + + // The issue workflow audits `error_issue.${type}` for every event type it + // attributes, so a new event type must not silently produce an undeclared action. + it("declares an `error_issue` verb for every issue event type", () => { + expect([...AuditResources.error_issue.verbs]).toEqual([...ErrorIssueEventType.literals]) + }) +}) + +describe("auditResourceFields", () => { + it("derives the resource type from the action", () => { + expect(auditResourceFields("alert_rule.created").resourceType).toBe("alert_rule") + expect(auditResourceFields("dashboard_share.rotated").resourceType).toBe("dashboard_share") + expect(auditResourceFields("dashboard.version_restored").resourceType).toBe("dashboard") + }) + + it("encodes the internal ID with the resource's own public prefix", () => { + const internal = "3f1b7c02-9a44-4d1e-8b2f-0c5d6e7a8b91" + const { resourceId } = auditResourceFields("alert_rule.created", internal) + expect(resourceId).toMatch(/^alrt_/) + expect(decodePublicId(PublicIdPrefixes.alertRule, resourceId!)).toBe(internal) + }) + + it("omits the resource ID for org-singleton resources", () => { + expect(auditResourceFields("ingest_key.rolled")).toEqual({ resourceType: "ingest_key" }) + expect(auditResourceFields("api.request")).toEqual({ resourceType: "api" }) + }) +}) diff --git a/apps/api/src/services/audit/audit-actions.ts b/apps/api/src/services/audit/audit-actions.ts new file mode 100644 index 000000000..47d3303a5 --- /dev/null +++ b/apps/api/src/services/audit/audit-actions.ts @@ -0,0 +1,95 @@ +import { encodePublicId, type PublicIdPrefix, PublicIdPrefixes } from "@maple/domain/http/v2" +import { ErrorIssueEventType } from "@maple/domain/http" + +/** + * Every audited action in Maple, grouped by the resource it acts on. + * + * The key is both the `resource_type` stored on the row and the `` + * half of the `.` action string, so the two can never disagree. + * `prefix` is the public-ID prefix the resource's internal ID is encoded with; + * resources that are org-singletons (`ingest_key`, `anomaly_settings`) or carry + * no resource at all (`api`) omit it, and passing a `resourceId` for one of + * those is a type error. + * + * Adding an entry here is what makes `record({ action: "." })` + * compile — a typo, or an action recorded before it is declared, fails the build. + */ +export const AuditResources = { + agent: { prefix: PublicIdPrefixes.actor, verbs: ["registered"] }, + alert_destination: { + prefix: PublicIdPrefixes.alertDestination, + verbs: ["created", "updated", "deleted"], + }, + alert_rule: { prefix: PublicIdPrefixes.alertRule, verbs: ["created", "updated", "deleted"] }, + anomaly_incident: { prefix: PublicIdPrefixes.anomalyIncident, verbs: ["resolved"] }, + /** Org-singleton settings — no resource id. */ + anomaly_settings: { verbs: ["updated"] }, + /** Refused requests, recorded by the auth layers; the route is in `metadata`. */ + api: { verbs: ["request"] }, + api_key: { prefix: PublicIdPrefixes.apiKey, verbs: ["created", "rolled", "revoked"] }, + attribute_mapping: { + prefix: PublicIdPrefixes.attributeMapping, + verbs: ["created", "updated", "deleted"], + }, + dashboard: { + prefix: PublicIdPrefixes.dashboard, + verbs: ["created", "updated", "deleted", "version_restored"], + }, + dashboard_share: { prefix: PublicIdPrefixes.dashboardShare, verbs: ["created", "rotated", "deleted"] }, + /** Verbs mirror the issue event types — `recordEvent` audits every one it attributes. */ + error_issue: { prefix: PublicIdPrefixes.errorIssue, verbs: ErrorIssueEventType.literals }, + /** Org-singleton public/private pair; which one rolled is in `metadata`. */ + ingest_key: { verbs: ["rolled"] }, + scrape_target: { prefix: PublicIdPrefixes.scrapeTarget, verbs: ["created", "updated", "deleted"] }, +} as const satisfies Record + +interface AuditResourceDefinition { + readonly prefix?: PublicIdPrefix + readonly verbs: ReadonlyArray +} + +export type AuditResourceType = keyof typeof AuditResources + +/** `.` for every declared pair — the closed set of audit actions. */ +export type AuditAction = { + [K in AuditResourceType]: `${K}.${(typeof AuditResources)[K]["verbs"][number]}` +}[AuditResourceType] + +type ResourceOf = A extends `${infer R}.${string}` + ? R extends AuditResourceType + ? R + : never + : never + +/** + * The `resourceId` option for an action: the resource's *internal* ID, encoded + * to its public `_…` form on the way to the row. Resources that declare + * no prefix (org-singletons) accept no `resourceId` at all. + */ +export type AuditResourceIdOption = (typeof AuditResources)[ResourceOf] extends { + readonly prefix: PublicIdPrefix +} + ? { readonly resourceId?: string } + : { readonly resourceId?: never } + +/** + * Derive the row's `resource_type` from the action and encode the internal + * resource ID into its public form, so no call site restates either. + */ +export const auditResourceFields = ( + action: AuditAction, + resourceId?: string, +): { readonly resourceType: AuditResourceType; readonly resourceId?: string } => { + // SAFETY: every `AuditAction` is built as `${resource}.${verb}` from the keys + // of `AuditResources`, so the segment before the dot is always one of them. + const resourceType = action.slice(0, action.indexOf(".")) as AuditResourceType + const resource = AuditResources[resourceType] + // Narrow rather than widen: org-singleton resources declare no `prefix` at all. + const prefix = "prefix" in resource ? resource.prefix : undefined + return { + resourceType, + ...(resourceId !== undefined && prefix !== undefined + ? { resourceId: encodePublicId(prefix, resourceId) } + : undefined), + } +} diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts index 2e97446c7..81521e52b 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts @@ -22,7 +22,6 @@ import { CLOSED_WORKFLOW_STATES, MACHINE_OWNED_WORKFLOW_STATES, } from "@maple/domain/http" -import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" import { actors, alertIncidents, @@ -443,8 +442,7 @@ const make: Effect.Effect< }, source: actor.type === "agent" ? "mcp" : "dashboard", action: `error_issue.${type}`, - resourceType: "error_issue", - resourceId: encodePublicId(PublicIdPrefixes.errorIssue, issueId), + resourceId: issueId, metadata: { ...(opts.fromState != null ? { from_state: opts.fromState } : undefined), ...(opts.toState != null ? { to_state: opts.toState } : undefined), From f83c29bfbac3fd47d7fc6593b772be436398bc7f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 30 Aug 2026 01:11:07 +0200 Subject: [PATCH 06/19] fix(audit): close the open follow-ups from the audit log review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Durability. The audit-events consumer had no dead letter queue and no final-attempt branch, so after five retries Cloudflare dropped the entry with nothing in the logs at the moment it happened. There is now an `audit-events-dlq` queue with no consumer — an entry landing there is a lost record and the point is that it survives — and the consumer logs the hand-off at Error, with the org and action read defensively off the body. It keeps retrying on the final attempt, because acking is what would discard the message instead of routing it. Attribution. The actors row knows who acted, never how, so every mutation reached through an API key or over MCP was recorded as a dashboard session — the MCP middleware set no audit reference at all. `CurrentAuditActor` now carries the surface alongside the credential, all four auth layers stamp it, and the issue-workflow mirror consults it instead of assuming. Maple's own sweeps run as an agent actor, which made auto-close and lease expiry read as a third-party agent over MCP; they are now recorded as `system`, which until today had no writer at all. Coverage. Audited org deletion, warehouse settings (updated, deleted, schema applied), the Slack and PlanetScale integration lifecycles including the metrics-token install, widget credential mint/revoke, investigations, and issue comments — which wrote their event row directly and so bypassed the audit mirror entirely. Secrets stay out: the entries record which credential was installed, never its value. Membership. Members are changed in Clerk, never through Maple's API, which is why `affected_user` had no writers. The Clerk receiver now audits `organizationMembership.*` against the member. Clerk's payload does not name the admin who acted, so the entry is attributed to `system` rather than guessing a user. Enabling the three events in the Clerk dashboard is what turns this on. UI. The list paginates by offset over a newest-first append-only table, so an entry written mid-scroll shifted later pages and made them repeat one row and skip another. The first Load more now pins `until` to the newest entry on screen, freezing the window, and pages are deduped by id on append. The header no longer claims to record "every change". The retention sweep's ctid-addressed delete already landed with the review fixes; verified rather than changed. --- apps/api/alchemy.run.ts | 8 ++ apps/api/src/audit-events-runtime.test.ts | 127 ++++++++++++++++++ apps/api/src/audit-events-runtime.ts | 64 ++++++++- apps/api/src/mcp/app.ts | 25 ++++ apps/api/src/mcp/lib/resolve-tenant.ts | 3 +- .../routes/v1/org-clickhouse-settings.http.ts | 22 ++- apps/api/src/routes/v1/organizations.http.ts | 8 +- apps/api/src/routes/v2/integrations.http.ts | 17 +++ apps/api/src/routes/v2/investigations.http.ts | 10 ++ .../src/routes/v2/widget-credentials.http.ts | 12 ++ apps/api/src/routes/webhooks/clerk.http.ts | 66 ++++++++- .../src/routes/webhooks/webhooks.http.test.ts | 67 ++++++++- .../services/audit/AuditLogService.test.ts | 67 ++++++++- .../api/src/services/audit/AuditLogService.ts | 19 +-- apps/api/src/services/audit/audit-actions.ts | 24 ++++ .../services/auth/ApiAuthorizationLayer.ts | 3 +- .../services/auth/ApiAuthorizationV2Layer.ts | 3 +- .../auth/SessionAuthorizationLayer.ts | 2 +- apps/api/src/services/auth/audit-actor.ts | 17 ++- .../errors/ErrorIssueWorkflowService.ts | 36 ++++- .../services/product-events/clerk-events.ts | 32 +++++ apps/api/wrangler.jsonc | 4 + .../components/settings/audit-log-section.tsx | 36 ++++- .../src/lib/services/atoms/audit-log-atoms.ts | 25 ++-- 24 files changed, 644 insertions(+), 53 deletions(-) create mode 100644 apps/api/src/audit-events-runtime.test.ts diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 162b5862a..e6b943f64 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -324,6 +324,11 @@ export const createMapleApi = ({ stage, domains, replayBlobs }: CreateMapleApiOp const auditEventsQueue = yield* Cloudflare.Queues.Queue("audit-events", { name: auditEventsQueueName, }) + // Parking lot for audit entries that exhausted their retries. Deliberately + // has no consumer: an entry landing here is a lost audit record, and the + // point is that it survives for inspection instead of being dropped. + const auditEventsDlqName = resolveWorkerName("audit-events-dlq", stage) + yield* Cloudflare.Queues.Queue("audit-events-dlq", { name: auditEventsDlqName }) const worker = (yield* Cloudflare.Worker("api", { name: resolveWorkerName("api", stage), @@ -441,6 +446,9 @@ export const createMapleApi = ({ stage, domains, replayBlobs }: CreateMapleApiOp yield* Cloudflare.Queues.Consumer("audit-events-consumer", { queueId: auditEventsQueue.queueId, scriptName: worker.workerName, + // `maxRetries` must stay in sync with AUDIT_EVENTS_MAX_RETRIES in + // audit-events-runtime.ts, which logs the drop on the final attempt. + deadLetterQueue: auditEventsDlqName, settings: { batchSize: 25, maxConcurrency: 2, diff --git a/apps/api/src/audit-events-runtime.test.ts b/apps/api/src/audit-events-runtime.test.ts new file mode 100644 index 000000000..072c9456a --- /dev/null +++ b/apps/api/src/audit-events-runtime.test.ts @@ -0,0 +1,127 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/primitives" +import { Effect, Layer, Schema } from "effect" +import { auditLogEntries } from "@maple/db" +import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { Database, DatabaseError } from "@/platform/DatabaseLive" +import { processAuditEventsBatch } from "./audit-events-runtime" +import { AuditLogEvent, encodeAuditLogEventSync } from "./services/audit/audit-event" + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const ORG = asOrgId("org_audit_consumer_test") +const createdDbs: TestDb[] = [] + +afterEach(() => cleanupTestDbs(createdDbs)) + +const event = (id: string) => + encodeAuditLogEventSync( + new AuditLogEvent({ + orgId: ORG, + id: Schema.decodeUnknownSync(AuditLogEvent.fields.id)(id), + actorType: "user", + source: "dashboard", + action: "dashboard.created", + outcome: "allowed", + occurredAtMs: 1_700_000_000_000, + }), + ) + +/** One queue message, recording which terminal call the consumer made on it. */ +const message = (body: unknown, attempts: number) => { + const calls: string[] = [] + return { + message: { + body, + attempts, + ack: () => calls.push("ack"), + retry: () => calls.push("retry"), + }, + calls, + } +} + +const run = (effect: Effect.Effect) => { + const db = createTestDb(createdDbs) + return effect.pipe(Effect.provide(db.layer)) +} + +const batchOf = (...messages: ReadonlyArray<{ readonly message: unknown }>) => + ({ messages: messages.map((entry) => entry.message) }) as never + +/** + * A database whose every write fails, so the consumer's retry path is exercised + * without depending on a real Postgres fault. + */ +const failingDatabase = Layer.succeed(Database, { + execute: () => + Effect.fail(new DatabaseError({ message: "insert failed", cause: new Error("insert failed") })), +}) + +describe("processAuditEventsBatch", () => { + it.effect("inserts a well-formed event and acks it", () => + run( + Effect.gen(function* () { + const first = message(event("11111111-1111-4111-8111-111111111111"), 1) + yield* processAuditEventsBatch(batchOf(first)) + + expect(first.calls).toEqual(["ack"]) + const database = yield* Database + const rows = yield* database.execute((db) => db.select().from(auditLogEntries)) + expect(rows.map((row) => row.action)).toEqual(["dashboard.created"]) + }), + ), + ) + + // Redelivery is expected — the queue retries whole batches — so a second + // delivery of an already-inserted event must be a no-op, not a duplicate row. + it.effect("is idempotent across redelivery of the same event", () => + run( + Effect.gen(function* () { + const body = event("22222222-2222-4222-8222-222222222222") + yield* processAuditEventsBatch(batchOf(message(body, 1))) + yield* processAuditEventsBatch(batchOf(message(body, 2))) + + const database = yield* Database + const rows = yield* database.execute((db) => db.select().from(auditLogEntries)) + expect(rows).toHaveLength(1) + }), + ), + ) + + // Cloudflare routes a message to the DLQ only when the consumer retries it + // past `max_retries`. Acking on the final attempt would discard the entry + // instead, which is exactly the silent drop this branch exists to prevent. + it.effect("retries a failed insert on the final attempt so the message reaches the DLQ", () => + Effect.gen(function* () { + const exhausted = message(event("33333333-3333-4333-8333-333333333333"), 6) + yield* processAuditEventsBatch(batchOf(exhausted)) + + expect(exhausted.calls).toEqual(["retry"]) + }).pipe(Effect.provide(failingDatabase)), + ) + + it.effect("retries a failed insert while attempts remain", () => + Effect.gen(function* () { + const failed = message(event("44444444-4444-4444-8444-444444444444"), 2) + yield* processAuditEventsBatch(batchOf(failed)) + + expect(failed.calls).toEqual(["retry"]) + }).pipe(Effect.provide(failingDatabase)), + ) + + // A message that cannot decode will never decode. Retrying only burns the + // attempts that would otherwise carry a recoverable message to the DLQ. + it.effect("acks a malformed message instead of retrying it forever", () => + run( + Effect.gen(function* () { + const malformed = message({ not: "an audit event" }, 1) + yield* processAuditEventsBatch(batchOf(malformed)) + + expect(malformed.calls).toEqual(["ack"]) + const database = yield* Database + const rows = yield* database.execute((db) => db.select().from(auditLogEntries)) + expect(rows).toEqual([]) + }), + ), + ) +}) diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts index b7b18d063..8e6c4a926 100644 --- a/apps/api/src/audit-events-runtime.ts +++ b/apps/api/src/audit-events-runtime.ts @@ -26,10 +26,33 @@ export const buildAuditEventsLayer = (_env: Record) => { export const flushAuditEventsTelemetry = (env: Record) => telemetry.flush(env) +/** + * Must match `maxRetries` on the audit-events consumer in `alchemy.run.ts` and + * `wrangler.jsonc`. Cloudflare routes the message to the DLQ after this many + * retries without telling us; the check below is what makes the hand-off + * visible in logs at the moment it happens. + */ +const AUDIT_EVENTS_MAX_RETRIES = 5 + +/** + * Best-effort identity for the exhaustion log. The body reached us as queue + * JSON and may be anything at all, so these read defensively rather than + * decoding — a drop must still be reported when the payload is the problem. + */ +const auditEventField = (body: unknown, field: string): string => { + if (typeof body !== "object" || body === null || !(field in body)) return "" + // SAFETY: `field in body` established the key exists on this object. + const value = (body as Record)[field] + return typeof value === "string" ? value : "" +} +const auditEventOrgId = (body: unknown) => auditEventField(body, "orgId") +const auditEventAction = (body: unknown) => auditEventField(body, "action") + /** * Audit events queue consumer: lowers each event to its `audit_log_entries` * row. The `(org_id, id)` primary key plus `onConflictDoNothing` makes queue - * redelivery idempotent; insert failures retry through the queue's policy. + * redelivery idempotent; insert failures retry through the queue's policy and, + * once exhausted, land in `audit-events-dlq` rather than disappearing. */ export const processAuditEventsBatch = (batch: MessageBatch) => Effect.gen(function* () { @@ -39,8 +62,11 @@ export const processAuditEventsBatch = (batch: MessageBatch) => (message) => decodeAuditLogEvent(message.body).pipe( Effect.matchEffect({ + // Undecodable now means undecodable on every redelivery, so retrying + // only burns attempts. Acked, but at Error: an audit entry that + // never reaches a row is lost evidence, not routine noise. onFailure: (error) => - Effect.logWarning("Discarding malformed audit event queue message").pipe( + Effect.logError("Discarding malformed audit event queue message").pipe( Effect.annotateLogs({ attempt: message.attempts, error: String(error) }), Effect.flatMap(() => Effect.sync(() => message.ack())), ), @@ -56,12 +82,36 @@ export const processAuditEventsBatch = (batch: MessageBatch) => yield* Effect.sync(() => message.ack()) }).pipe( Effect.withSpan("auditEvents.processMessage"), - Effect.catchCause((cause) => - Effect.logWarning("Audit event insert failed; retrying").pipe( - Effect.annotateLogs({ attempt: message.attempts, error: String(cause) }), + Effect.catchCause((cause) => { + // Retrying past the limit is what hands the message to the + // DLQ; acking here would silently discard it instead. + const isFinalAttempt = message.attempts > AUDIT_EVENTS_MAX_RETRIES + const outcome = isFinalAttempt ? "exhausted_dlq" : "retry" + return Effect.annotateCurrentSpan({ + "audit.queue.message.outcome": outcome, + }).pipe( + Effect.flatMap(() => + isFinalAttempt + ? Effect.logError( + "Audit event exhausted retries; routed to dead letter queue", + ).pipe( + Effect.annotateLogs({ + attempt: message.attempts, + orgId: auditEventOrgId(message.body), + action: auditEventAction(message.body), + error: String(cause), + }), + ) + : Effect.logWarning("Audit event insert failed; retrying").pipe( + Effect.annotateLogs({ + attempt: message.attempts, + error: String(cause), + }), + ), + ), Effect.flatMap(() => Effect.sync(() => message.retry())), - ), - ), + ) + }), ), }), ), diff --git a/apps/api/src/mcp/app.ts b/apps/api/src/mcp/app.ts index 867536a2d..c7abd49cb 100644 --- a/apps/api/src/mcp/app.ts +++ b/apps/api/src/mcp/app.ts @@ -10,6 +10,8 @@ import { InstructionsResource } from "./resources/instructions" import { sessionStore } from "./lib/session-store" import type { McpToolExecutor } from "./dispatcher" import { CurrentMcpRequestTenant, CurrentMcpTenant, resolveHttpMcpTenant } from "./lib/query-warehouse" +import { type AuditActorInfo, CurrentAuditActor } from "@/services/auth/audit-actor" +import { INTERNAL_SERVICE_PREFIX } from "./lib/resolve-tenant" import { ApiKeysService } from "@/services/org/ApiKeysService" import { AuthService } from "@/services/auth/AuthService" import { Env } from "@/platform/Env" @@ -93,6 +95,23 @@ const mcpUnavailable = () => ), ) +/** + * Which credential an MCP request presented, as far as the transport can tell. + * Mirrors the branches in `resolveMcpTenantContext`: an internal service token + * is Maple acting on its own behalf, any other bearer is an API key or OAuth + * token, and no bearer at all means a forwarded dashboard session. + */ +const mcpAuditActor = (headers: Record): AuditActorInfo => { + const authorization = headers["authorization"] ?? headers["Authorization"] + if (authorization?.toLowerCase().startsWith("bearer ") !== true) { + return { type: "user", source: "mcp" } + } + const bearer = authorization.slice("bearer ".length).trim() + return bearer.startsWith(INTERNAL_SERVICE_PREFIX) + ? { type: "system", source: "system" } + : { type: "api_key", source: "mcp" } +} + const McpAuthorizationMiddleware = HttpRouter.middleware<{ provides: CurrentMcpTenant }>()( Effect.gen(function* () { const apiKeys = yield* ApiKeysService @@ -108,6 +127,12 @@ const McpAuthorizationMiddleware = HttpRouter.middleware<{ provides: CurrentMcpT Effect.flatMap((tenant) => Effect.provideService(httpEffect, CurrentMcpTenant, tenant).pipe( Effect.provideService(CurrentMcpRequestTenant, tenant), + // Without this an MCP mutation reads the reference's `undefined` + // default and is audited as a dashboard session. The credential + // kind is all this layer can see — `resolveMcpTenantContext` + // returns the tenant, not the key it resolved — so the key id is + // deliberately absent rather than guessed. + Effect.provideService(CurrentAuditActor, mcpAuditActor(request.headers)), ), ), Effect.catchTags({ diff --git a/apps/api/src/mcp/lib/resolve-tenant.ts b/apps/api/src/mcp/lib/resolve-tenant.ts index 3e7168602..993877c08 100644 --- a/apps/api/src/mcp/lib/resolve-tenant.ts +++ b/apps/api/src/mcp/lib/resolve-tenant.ts @@ -14,7 +14,8 @@ import { import { recordExpectedMcpFailure } from "@/mcp/expected-failures" import { sessionStore } from "@/mcp/lib/session-store" -const INTERNAL_SERVICE_PREFIX = "maple_svc_" +/** Exported so the audit layer classifies the same token the same way. */ +export const INTERNAL_SERVICE_PREFIX = "maple_svc_" const decodeOrgId = Schema.decodeUnknownEffect(OrgId) const decodeUserId = Schema.decodeUnknownEffect(UserId) const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) diff --git a/apps/api/src/routes/v1/org-clickhouse-settings.http.ts b/apps/api/src/routes/v1/org-clickhouse-settings.http.ts index 79e1f2857..6d43e4b8f 100644 --- a/apps/api/src/routes/v1/org-clickhouse-settings.http.ts +++ b/apps/api/src/routes/v1/org-clickhouse-settings.http.ts @@ -1,6 +1,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant, MapleApi } from "@maple/domain/http" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( @@ -20,7 +21,18 @@ export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( .handle("upsert", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* service.upsert(tenant.orgId, tenant.userId, tenant.roles, payload) + const updated = yield* service.upsert( + tenant.orgId, + tenant.userId, + tenant.roles, + payload, + ) + // URL/user/database identify the connection; the password in the + // payload is write-only and never reaches an audit row. + yield* recordHttpAudit("warehouse_settings.updated", { + metadata: { url: payload.url, user: payload.user, database: payload.database }, + }) + return updated }), ) .handle("schemaDiff", () => @@ -32,7 +44,9 @@ export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( .handle("applySchema", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* service.applySchema(tenant.orgId, tenant.userId, tenant.roles) + const applied = yield* service.applySchema(tenant.orgId, tenant.userId, tenant.roles) + yield* recordHttpAudit("warehouse_settings.schema_applied") + return applied }), ) .handle("applySchemaStatus", () => @@ -50,7 +64,9 @@ export const HttpOrgClickHouseSettingsLive = HttpApiBuilder.group( .handle("delete", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* service.delete(tenant.orgId, tenant.roles) + const deleted = yield* service.delete(tenant.orgId, tenant.roles) + yield* recordHttpAudit("warehouse_settings.deleted") + return deleted }), ) }), diff --git a/apps/api/src/routes/v1/organizations.http.ts b/apps/api/src/routes/v1/organizations.http.ts index 231b7ce75..6c427c446 100644 --- a/apps/api/src/routes/v1/organizations.http.ts +++ b/apps/api/src/routes/v1/organizations.http.ts @@ -1,6 +1,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant, MapleApi } from "@maple/domain/http" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { OrganizationService } from "@/services/org/OrganizationService" export const HttpOrganizationsLive = HttpApiBuilder.group(MapleApi, "organizations", (handlers) => @@ -10,7 +11,12 @@ export const HttpOrganizationsLive = HttpApiBuilder.group(MapleApi, "organizatio return handlers.handle("delete", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - return yield* organizationService.delete(tenant.orgId, tenant.roles) + const deleted = yield* organizationService.delete(tenant.orgId, tenant.roles) + // Recorded after the fact so a refused delete cannot leave an entry + // claiming the org is gone. The row outlives the org: nothing + // cascades `audit_log_entries`, which is the point of a trail. + yield* recordHttpAudit("organization.deleted") + return deleted }), ) }), diff --git a/apps/api/src/routes/v2/integrations.http.ts b/apps/api/src/routes/v2/integrations.http.ts index 22d2f41dd..f5fb66e7c 100644 --- a/apps/api/src/routes/v2/integrations.http.ts +++ b/apps/api/src/routes/v2/integrations.http.ts @@ -31,6 +31,7 @@ import { V2TimeRangeInvalid, } from "@maple/domain/http/v2" import { Array as Arr, Effect, Option } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { requireAdmin } from "@/services/auth/auth" import { Env } from "@/platform/Env" import { EdgeCacheService } from "@maple/cache" @@ -258,6 +259,7 @@ export const HttpV2SlackIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "sla const result = yield* slack .startInstall(tenant.orgId, tenant.userId, callbackUrl) .pipe(tapHttpErrors("Slack install failed")) + yield* recordHttpAudit("slack_integration.install_started") return { object: "slack_integration.install" as const, url: result.url, @@ -273,6 +275,7 @@ export const HttpV2SlackIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "sla yield* slack .uninstall(tenant.orgId) .pipe(tapHttpErrors("Slack integration uninstall failed")) + yield* recordHttpAudit("slack_integration.uninstalled") return { object: "slack_integration" as const, installed: false as const, @@ -353,6 +356,7 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( returnTo: payload.return_to, }) .pipe(tapHttpErrors("PlanetScale connect failed")) + yield* recordHttpAudit("planetscale_integration.connect_started") return { object: "planetscale_integration.connect" as const, redirect_url: result.redirectUrl, @@ -397,6 +401,13 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( excludeBranches: payload.exclude_branches, }) .pipe(tapHttpErrors("PlanetScale organization selection failed")) + yield* recordHttpAudit("planetscale_integration.organization_selected", { + metadata: { + organization: payload.organization, + include_branches: payload.include_branches, + exclude_branches: payload.exclude_branches, + }, + }) return toPlanetScaleStatus(status) }), ) @@ -414,6 +425,11 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( tokenSecret: payload.token_secret, }) .pipe(tapHttpErrors("PlanetScale metrics token update failed")) + // The token id names which credential was installed; its secret + // is write-only and never reaches an audit row. + yield* recordHttpAudit("planetscale_integration.metrics_token_set", { + metadata: { token_id: payload.token_id }, + }) return toPlanetScaleStatus(status) }), ) @@ -426,6 +442,7 @@ export const HttpV2PlanetScaleIntegrationsLive = HttpApiBuilder.group( yield* planetscale .disconnect(tenant.orgId) .pipe(tapHttpErrors("PlanetScale disconnect failed")) + yield* recordHttpAudit("planetscale_integration.disconnected") return { object: "planetscale_integration" as const, connected: false as const, diff --git a/apps/api/src/routes/v2/investigations.http.ts b/apps/api/src/routes/v2/investigations.http.ts index 54388f0d7..31fed9372 100644 --- a/apps/api/src/routes/v2/investigations.http.ts +++ b/apps/api/src/routes/v2/investigations.http.ts @@ -21,6 +21,7 @@ import type { V2InvestigationSubject, } from "@maple/domain/http/v2" import { Effect, Match, Schema } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { InvestigationService } from "@/services/errors/InvestigationService" const toWireSubject = Effect.fn("HttpV2Investigations.toWireSubject")(function* ( @@ -265,6 +266,10 @@ export const HttpV2InvestigationsLive = HttpApiBuilder.group(MapleApiV2, "invest : undefined), }), ) + yield* recordHttpAudit("investigation.created", { + resourceId: doc.id, + metadata: { subject_type: payload.subject.type }, + }) return yield* serializeInvestigation(doc) }), @@ -273,6 +278,7 @@ export const HttpV2InvestigationsLive = HttpApiBuilder.group(MapleApiV2, "invest Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const doc = yield* service.restartInvestigation(tenant.orgId, params.id) + yield* recordHttpAudit("investigation.restarted", { resourceId: doc.id }) return yield* serializeInvestigation(doc) }), @@ -281,6 +287,10 @@ export const HttpV2InvestigationsLive = HttpApiBuilder.group(MapleApiV2, "invest Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const doc = yield* service.updateStatus(tenant.orgId, params.id, payload.status) + yield* recordHttpAudit("investigation.status_changed", { + resourceId: doc.id, + metadata: { to_status: payload.status }, + }) return yield* serializeInvestigation(doc) }), diff --git a/apps/api/src/routes/v2/widget-credentials.http.ts b/apps/api/src/routes/v2/widget-credentials.http.ts index 12cc2a84c..4cd9a00ca 100644 --- a/apps/api/src/routes/v2/widget-credentials.http.ts +++ b/apps/api/src/routes/v2/widget-credentials.http.ts @@ -2,6 +2,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant } from "@maple/domain/http" import { MapleApiV2, isoTimestamp } from "@maple/domain/http/v2" import { Effect } from "effect" +import { recordHttpAudit } from "@/services/audit/AuditLogService" import { ApiKeysService } from "@/services/org/ApiKeysService" /** @@ -49,6 +50,14 @@ export const HttpV2WidgetCredentialsLive = HttpApiBuilder.group(MapleApiV2, "wid // letting it resolve with the API-key default — is `root`. roles: tenant.roles, }) + // A credential mint is the security event; the secret itself never + // reaches the row, only which installation it was issued to. + yield* recordHttpAudit("widget_credential.minted", { + metadata: { + installation_id: params.installation_id, + scopes: credential.scopes ?? WIDGET_CREDENTIAL_SCOPES, + }, + }) return { object: "widget_credential" as const, secret: credential.secret, @@ -68,6 +77,9 @@ export const HttpV2WidgetCredentialsLive = HttpApiBuilder.group(MapleApiV2, "wid // to revoke: this is the sign-out path, and an error the app // cannot act on while signing out anyway is worse than silence. yield* apiKeys.revokeDeviceKeys(tenant.orgId, params.installation_id) + yield* recordHttpAudit("widget_credential.revoked", { + metadata: { installation_id: params.installation_id }, + }) return { object: "widget_credential" as const, deleted: true as const } }), ) diff --git a/apps/api/src/routes/webhooks/clerk.http.ts b/apps/api/src/routes/webhooks/clerk.http.ts index e84f5543d..4f27fafca 100644 --- a/apps/api/src/routes/webhooks/clerk.http.ts +++ b/apps/api/src/routes/webhooks/clerk.http.ts @@ -1,25 +1,51 @@ -import { Effect, Option } from "effect" +import { Effect, Option, Schema } from "effect" import { HttpRouter, type HttpServerRequest } from "effect/unstable/http" import { Env } from "@/platform/Env" import { + CLERK_MEMBERSHIP_EVENTS, decodeClerkEnvelope, + decodeClerkOrganizationMembership, decodeClerkUserCreated, + isClerkMembershipEvent, signupCompletedEvent, } from "@/services/product-events/clerk-events" +import type { ClerkOrganizationMembershipData } from "@/services/product-events/clerk-events" import { ProductEventsService } from "@/services/product-events/ProductEventsService" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { OrgId, UserId } from "@maple/domain/primitives" import { receiveSvixWebhook, webhookText } from "./svix-receiver" /** - * Clerk webhook receiver: `user.created` → `signup_completed` product event. - * Public route; authenticity is the Svix signature (`CLERK_WEBHOOK_SECRET`). - * Any other event type is acknowledged with 200 so Clerk does not retry it. + * Clerk webhook receiver: `user.created` → `signup_completed` product event, + * and `organizationMembership.*` → an org audit entry. Public route; + * authenticity is the Svix signature (`CLERK_WEBHOOK_SECRET`). Any other event + * type is acknowledged with 200 so Clerk does not retry it. + * + * Membership is the one org change the web app makes in Clerk rather than + * through Maple's API, so this receiver is the only writer of `affected_user`. + * Enabling the three `organizationMembership.*` events in the Clerk dashboard + * is what turns it on — until then Clerk simply never delivers them. */ const ROUTE = "/webhooks/clerk" +const decodeOrgId = Schema.decodeUnknownEffect(OrgId) +const decodeUserId = Schema.decodeUnknownEffect(UserId) + +/** + * Brand the two Clerk IDs together so a payload with either one malformed is + * dropped whole, rather than recording an entry against a half-known subject. + */ +const decodeMembershipIds = (data: ClerkOrganizationMembershipData) => + Effect.all({ + orgId: decodeOrgId(data.organization.id), + userId: decodeUserId(data.public_user_data.user_id), + }) + export const ClerkWebhookRouter = HttpRouter.use((router) => Effect.gen(function* () { const env = yield* Env const productEvents = yield* ProductEventsService + const audit = yield* AuditLogService const handle = Effect.fn("ClerkWebhook.receive")(function* ( req: HttpServerRequest.HttpServerRequest, @@ -59,6 +85,38 @@ export const ClerkWebhookRouter = HttpRouter.use((router) => } else { yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "parse_rejected" }) } + } else if (isClerkMembershipEvent(envelope.value.type)) { + const membership = yield* decodeClerkOrganizationMembership(envelope.value.data).pipe( + Effect.tapError((error) => + Effect.logInfo("Clerk membership payload failed to decode").pipe( + Effect.annotateLogs({ event: envelope.value.type, error: String(error) }), + ), + ), + Effect.option, + ) + if (Option.isSome(membership)) { + const ids = yield* decodeMembershipIds(membership.value).pipe(Effect.option) + if (Option.isSome(ids)) { + // Clerk's payload names the member, never the admin who acted, so + // attributing this to a user would be a guess. `system` says + // truthfully that Maple learned of the change rather than made it. + yield* audit.record({ + orgId: ids.value.orgId, + actor: { type: "system" }, + source: "system", + action: `member.${CLERK_MEMBERSHIP_EVENTS[envelope.value.type]}`, + affectedUserId: ids.value.userId, + ...(membership.value.role !== undefined + ? { metadata: { role: membership.value.role } } + : undefined), + }) + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "handled" }) + } else { + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "parse_rejected" }) + } + } else { + yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "parse_rejected" }) + } } else { yield* Effect.annotateCurrentSpan({ "maple.webhook.outcome": "ignored" }) } diff --git a/apps/api/src/routes/webhooks/webhooks.http.test.ts b/apps/api/src/routes/webhooks/webhooks.http.test.ts index f130cfb53..cd825d920 100644 --- a/apps/api/src/routes/webhooks/webhooks.http.test.ts +++ b/apps/api/src/routes/webhooks/webhooks.http.test.ts @@ -4,6 +4,7 @@ import { HttpRouter } from "effect/unstable/http" import { Env } from "@/platform/Env" import { ProductEventsService, type ProductEventInput } from "@/services/product-events/ProductEventsService" import { signSvix } from "@/services/product-events/svix" +import { AuditLogService, type AuditLogRecordInput } from "@/services/audit/AuditLogService" import { AutumnWebhookRouter } from "./autumn.http" import { ClerkWebhookRouter } from "./clerk.http" @@ -34,11 +35,27 @@ const recordingProductEvents = () => { return { tracked, layer } } +const recordingAudit = () => { + const recorded: Array = [] + const layer = Layer.succeed(AuditLogService, { + record: (input) => Effect.sync(() => void recorded.push(input)), + list: () => Effect.succeed([]), + }) + return { recorded, layer } +} + const makeRouterLayer = ( router: typeof ClerkWebhookRouter, config: Record, productEvents: Layer.Layer, -) => router.pipe(Layer.provide(productEvents), Layer.provide(Env.layer), Layer.provide(makeConfig(config))) + audit: Layer.Layer = recordingAudit().layer, +) => + router.pipe( + Layer.provide(productEvents), + Layer.provide(audit), + Layer.provide(Env.layer), + Layer.provide(makeConfig(config)), + ) const signedHeaders = (secret: string, body: string, nowMs: number, id = "msg_test") => Effect.gen(function* () { @@ -118,7 +135,55 @@ const AUTUMN_BILLING_UPDATED = JSON.stringify({ }, }) +const CLERK_MEMBERSHIP_CREATED = JSON.stringify({ + type: "organizationMembership.created", + timestamp: 1_700_000_000_000, + data: { + organization: { id: "org_42" }, + public_user_data: { user_id: "user_2abc" }, + role: "org:admin", + }, +}) + describe("ClerkWebhookRouter", () => { + // Membership is changed in Clerk, never through Maple's API, so this receiver + // is the only writer of `affected_user`. + it.effect("audits an organizationMembership.created delivery against the member", () => + Effect.gen(function* () { + const events = recordingProductEvents() + const audit = recordingAudit() + const configured = HttpRouter.toWebHandler( + makeRouterLayer( + ClerkWebhookRouter, + { CLERK_WEBHOOK_SECRET: CLERK_SECRET }, + events.layer, + audit.layer, + ), + { disableLogger: true }, + ) + yield* Effect.gen(function* () { + const now = Date.now() + const headers = yield* signedHeaders(CLERK_SECRET, CLERK_MEMBERSHIP_CREATED, now) + const response = yield* post( + configured.handler, + "/webhooks/clerk", + CLERK_MEMBERSHIP_CREATED, + headers, + ) + assert.strictEqual(response.status, 200) + assert.strictEqual(audit.recorded.length, 1) + const entry = audit.recorded[0]! + assert.strictEqual(entry.action, "member.added") + assert.strictEqual(entry.affectedUserId, "user_2abc") + assert.strictEqual(entry.orgId, "org_42") + // Clerk's payload never names the admin who acted; claiming a user + // here would be a guess, so the entry is Maple recording what it learned. + assert.strictEqual(entry.actor.type, "system") + assert.strictEqual(entry.source, "system") + }).pipe(Effect.ensuring(Effect.promise(() => configured.dispose()))) + }), + ) + it.effect( "503s while unconfigured, 401s a bad signature, and emits signup_completed for user.created", () => diff --git a/apps/api/src/services/audit/AuditLogService.test.ts b/apps/api/src/services/audit/AuditLogService.test.ts index 543d16948..ddd3a8ca2 100644 --- a/apps/api/src/services/audit/AuditLogService.test.ts +++ b/apps/api/src/services/audit/AuditLogService.test.ts @@ -5,7 +5,10 @@ import { OrgId, UserId } from "@maple/domain/primitives" import { Effect, Layer, Schema } from "effect" import { TestClock } from "effect/testing" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" -import { AuditLogService } from "./AuditLogService" +import { AuditLogService, recordHttpAudit } from "./AuditLogService" +import { CurrentTenant } from "@maple/domain/http" +import { ApiKeyId } from "@maple/domain/primitives" +import { type AuditActorInfo, CurrentAuditActor } from "@/services/auth/audit-actor" const asOrgId = Schema.decodeUnknownSync(OrgId) const asUserId = Schema.decodeUnknownSync(UserId) @@ -17,6 +20,7 @@ const createdDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(createdDbs)) const DASHBOARD_ID = "3f1b7c02-9a44-4d1e-8b2f-0c5d6e7a8b91" +const API_KEY = Schema.decodeUnknownSync(ApiKeyId)("7b2e4c10-55aa-4d3e-9f21-1a2b3c4d5e6f") const makeLayer = () => AuditLogService.layer.pipe(Layer.provide(createTestDb(createdDbs).layer)) @@ -186,6 +190,67 @@ describe("AuditLogService", () => { ) }) + // The credential and the surface are the two facts a mutation handler cannot + // re-derive, and getting them wrong is what made API-key and MCP actions read + // back as dashboard sessions. + describe("recordHttpAudit attribution", () => { + const tenant = new CurrentTenant.TenantSchema({ + orgId: ORG, + userId: USER, + roles: [], + authMode: "self_hosted", + }) + + const recordAs = (info: AuditActorInfo | undefined) => + Effect.gen(function* () { + const audit = yield* AuditLogService + yield* recordHttpAudit("dashboard.created", { resourceId: DASHBOARD_ID }) + const rows = yield* audit.list(ORG, { limit: 1, offset: 0 }) + return rows[0]! + }).pipe( + Effect.provideService(CurrentTenant.Context, tenant), + Effect.provideService(CurrentAuditActor, info), + Effect.provide(makeLayer().pipe(Layer.provide(Layer.succeed(WorkerEnvironment, {})))), + ) + + it.effect("attributes an API-key request to the key, not the dashboard", () => + Effect.gen(function* () { + const row = yield* recordAs({ type: "api_key", apiKeyId: API_KEY, source: "api" }) + expect(row.actorType).toBe("api_key") + expect(row.source).toBe("api") + expect(row.apiKeyId).toBe(API_KEY) + }), + ) + + it.effect("records the MCP surface rather than assuming a dashboard session", () => + Effect.gen(function* () { + const row = yield* recordAs({ type: "api_key", source: "mcp" }) + expect(row.source).toBe("mcp") + expect(row.actorType).toBe("api_key") + }), + ) + + it.effect("records Maple's own internal-token actions as system", () => + Effect.gen(function* () { + const row = yield* recordAs({ type: "system", source: "system" }) + expect(row.actorType).toBe("system") + expect(row.source).toBe("system") + }), + ) + + // Requests that skipped every auth middleware still have a tenant; the + // fallback must not invent a credential it did not see. + it.effect("falls back to the tenant user when no middleware set the reference", () => + Effect.gen(function* () { + const row = yield* recordAs(undefined) + expect(row.actorType).toBe("user") + expect(row.source).toBe("dashboard") + expect(row.userId).toBe(USER) + expect(row.apiKeyId).toBeNull() + }), + ) + }) + it.effect("writes directly when the queue binding is absent from the worker environment", () => Effect.gen(function* () { const audit = yield* AuditLogService diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index 8ebc4110d..fbce6c231 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -286,10 +286,11 @@ const requestContext = Effect.gen(function* () { /** * Record an audit entry for the current authenticated HTTP request, deriving - * the actor from the tenant plus the auth middleware's `CurrentAuditActor`, - * and request forensics (request id, origin) from the Cloudflare headers. - * Session requests (and requests that bypassed the standard middlewares) - * attribute to the user; API-key requests attribute to the key. + * the actor and surface from the tenant plus the auth middleware's + * `CurrentAuditActor`, and request forensics (request id, origin) from the + * Cloudflare headers. The credential kind and the surface both come from the + * reference — an API-key or MCP request must not read back as a dashboard + * session. */ export const recordHttpAudit = ( action: A, @@ -304,15 +305,17 @@ export const recordHttpAudit = ( const tenant = yield* CurrentTenant.Context const info = yield* CurrentAuditActor const context = yield* requestContext - const isApiKey = info?.type === "api_key" + // No reference means the request bypassed every auth middleware (internal + // tokens, tests). Attribute to the tenant's user rather than inventing a + // credential, but do not claim a surface the request may not have used. yield* audit.record({ orgId: tenant.orgId, actor: { - type: isApiKey ? "api_key" : "user", + type: info?.type ?? "user", userId: tenant.userId, - ...(isApiKey && info.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), + ...(info?.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), }, - source: isApiKey ? "api" : "dashboard", + source: info?.source ?? "dashboard", action, ...context, ...opts, diff --git a/apps/api/src/services/audit/audit-actions.ts b/apps/api/src/services/audit/audit-actions.ts index 47d3303a5..444284d52 100644 --- a/apps/api/src/services/audit/audit-actions.ts +++ b/apps/api/src/services/audit/audit-actions.ts @@ -40,7 +40,31 @@ export const AuditResources = { error_issue: { prefix: PublicIdPrefixes.errorIssue, verbs: ErrorIssueEventType.literals }, /** Org-singleton public/private pair; which one rolled is in `metadata`. */ ingest_key: { verbs: ["rolled"] }, + investigation: { prefix: PublicIdPrefixes.investigation, verbs: ["created", "restarted", "status_changed"] }, + /** + * Org-singleton connections. `*_started` is the admin action Maple sees; the + * OAuth round trip completes at the provider's callback. + */ + planetscale_integration: { + verbs: ["connect_started", "organization_selected", "metrics_token_set", "disconnected"], + }, + slack_integration: { verbs: ["install_started", "uninstalled"] }, + /** + * Org membership, learned from Clerk's webhook — the web app changes members + * in Clerk directly, so nothing reaches Maple's own API. The member is the + * entry's `affected_user`; no prefix, since Clerk IDs are already public. + */ + member: { verbs: ["added", "role_changed", "removed"] }, + /** + * The org itself. No prefix: every row already carries `org_id`, and a + * deleted org has no public ID left to resolve. + */ + organization: { verbs: ["deleted"] }, scrape_target: { prefix: PublicIdPrefixes.scrapeTarget, verbs: ["created", "updated", "deleted"] }, + /** Org-singleton BYO-ClickHouse connection; holds warehouse credentials. */ + warehouse_settings: { verbs: ["updated", "deleted", "schema_applied"] }, + /** Short-lived device credentials for the mobile widget; keyed by installation. */ + widget_credential: { verbs: ["minted", "revoked"] }, } as const satisfies Record interface AuditResourceDefinition { diff --git a/apps/api/src/services/auth/ApiAuthorizationLayer.ts b/apps/api/src/services/auth/ApiAuthorizationLayer.ts index 27334c2be..6c57bf3ac 100644 --- a/apps/api/src/services/auth/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationLayer.ts @@ -85,6 +85,7 @@ export const ApiAuthorizationLayer = Layer.effect( Effect.provideService(CurrentAuditActor, { type: "api_key", apiKeyId: resolved.keyId, + source: "api", }), ) } @@ -93,7 +94,7 @@ export const ApiAuthorizationLayer = Layer.effect( yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) return yield* httpEffect.pipe( Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), - Effect.provideService(CurrentAuditActor, { type: "user" }), + Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), ) }), }) diff --git a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts index b203095a1..b818b8429 100644 --- a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts @@ -173,6 +173,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( Effect.provideService(CurrentAuditActor, { type: "api_key", apiKeyId: resolved.keyId, + source: "api", }), ) } @@ -185,7 +186,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) return yield* httpEffect.pipe( Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), - Effect.provideService(CurrentAuditActor, { type: "user" }), + Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), ) }), }) diff --git a/apps/api/src/services/auth/SessionAuthorizationLayer.ts b/apps/api/src/services/auth/SessionAuthorizationLayer.ts index 82a706283..6c2df1fce 100644 --- a/apps/api/src/services/auth/SessionAuthorizationLayer.ts +++ b/apps/api/src/services/auth/SessionAuthorizationLayer.ts @@ -50,7 +50,7 @@ export const SessionAuthorizationLayer = Layer.effect( yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) return yield* httpEffect.pipe( Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), - Effect.provideService(CurrentAuditActor, { type: "user" }), + Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), ) }), }) diff --git a/apps/api/src/services/auth/audit-actor.ts b/apps/api/src/services/auth/audit-actor.ts index 5258a336b..43147417e 100644 --- a/apps/api/src/services/auth/audit-actor.ts +++ b/apps/api/src/services/auth/audit-actor.ts @@ -1,21 +1,28 @@ import { Context } from "effect" +import type { AuditLogSource } from "@maple/domain/http" import type { ApiKeyId } from "@maple/domain/primitives" /** - * How the current HTTP request authenticated, for audit attribution. The - * tenant context deliberately does not say whether a request came from a - * dashboard session or an API key — this reference carries that one fact. + * How the current request authenticated, for audit attribution. The tenant + * context deliberately does not say whether a request came from a dashboard + * session, an API key, or MCP — this reference carries those two facts, which + * nothing downstream can re-derive. */ export interface AuditActorInfo { - readonly type: "user" | "api_key" + /** `system` is Maple itself acting through an internal service token. */ + readonly type: "user" | "api_key" | "system" readonly apiKeyId?: ApiKeyId + /** The surface the request arrived through, recorded as the entry's `source`. */ + readonly source: AuditLogSource } /** * A reference (typed default, no handler requirement) rather than a service: * the auth middlewares override it per request, and handlers that never record * audit entries are unaffected. `undefined` means the request skipped the - * standard auth middlewares (internal tokens, tests). + * standard auth middlewares (internal tokens, queue consumers, crons) — callers + * must then fall back to whatever attribution they can establish themselves, + * never assume a dashboard session. */ export class CurrentAuditActor extends Context.Reference( "@maple/api/services/auth/CurrentAuditActor", diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts index 81521e52b..27b4c837f 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts @@ -39,6 +39,8 @@ import { and, desc, eq, inArray, sql } from "drizzle-orm" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import { Database } from "@/platform/DatabaseLive" import { AuditLogService } from "@/services/audit/AuditLogService" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { SYSTEM_ERRORS_AGENT_NAME } from "@/services/auth/system-actors" import { readTxid, txidColumn } from "@/platform/electric-txid" import { dateToMs, msToDate } from "@/platform/time" import { ErrorActorsService } from "./ErrorActorsService" @@ -421,12 +423,22 @@ const make: Effect.Effect< ) const actor = rows[0] if (actor === undefined || (actor.type !== "agent" && actor.type !== "user")) return + // Maple's own sweeps run as an agent actor (`ensureSystemActor` mints + // one), so without this check auto-close, lease expiry and fix + // verification all read as a third-party agent acting over MCP. + const isSystemActor = actor.type === "agent" && actor.agentName === SYSTEM_ERRORS_AGENT_NAME + // The actors row knows *who*, never *how*: it is the same row whether + // the mutation arrived from the dashboard, an API key, or MCP. The + // request's `CurrentAuditActor` is the only thing that knows the + // credential and surface, so a human actor is attributed through it and + // falls back to a dashboard session only when nothing set it (queue + // consumers, crons). + const request = yield* CurrentAuditActor yield* audit.record({ orgId, - // A human actor at this layer may have acted from the dashboard or - // over MCP — the issue event does not say which. - actor: - actor.type === "agent" + actor: isSystemActor + ? { type: "system", actorId, label: SYSTEM_ERRORS_AGENT_NAME } + : actor.type === "agent" ? { type: "agent", actorId, @@ -436,11 +448,18 @@ const make: Effect.Effect< ...(actor.createdBy === null ? undefined : { userId: actor.createdBy }), } : { - type: "user", + type: request?.type ?? "user", ...(actor.userId === null ? undefined : { userId: actor.userId }), + ...(request?.apiKeyId === undefined + ? undefined + : { apiKeyId: request.apiKeyId }), actorId, }, - source: actor.type === "agent" ? "mcp" : "dashboard", + source: isSystemActor + ? "system" + : actor.type === "agent" + ? "mcp" + : (request?.source ?? "dashboard"), action: `error_issue.${type}`, resourceId: issueId, metadata: { @@ -859,6 +878,11 @@ const make: Effect.Effect< createdAt: msToDate(timestamp), } yield* dbExecute((db) => db.insert(errorIssueEvents).values(row)) + // This path writes the event row itself rather than going through + // `recordEvent`, so the audit mirror has to be invoked explicitly. The + // comment body stays out of the row — the audit records that a comment + // was made, not what it said. + yield* recordEventAudit(orgId, issueId, actorId, type, {}) yield* actorsService.touchActor(orgId, actorId, timestamp) const actorMap = yield* actorsService.collectActorDocs(orgId, [actorId]) return rowToEvent(row, actorMap) diff --git a/apps/api/src/services/product-events/clerk-events.ts b/apps/api/src/services/product-events/clerk-events.ts index c193820ef..b6f61bdab 100644 --- a/apps/api/src/services/product-events/clerk-events.ts +++ b/apps/api/src/services/product-events/clerk-events.ts @@ -29,6 +29,38 @@ export const ClerkUserCreatedData = Schema.Struct({ created_at: Schema.optionalKey(Schema.Number), }) +/** + * `organizationMembership.*` payload. Membership is managed in Clerk directly — + * the web app never asks Maple's API to add or remove a member — so this + * webhook is the only place those changes can be audited. Clerk does not name + * the admin who made the change in this payload, only the member it happened + * to, which is why the resulting entries are attributed to `system`. + */ +export const ClerkOrganizationMembershipData = Schema.Struct({ + organization: Schema.Struct({ id: Schema.String }), + public_user_data: Schema.Struct({ user_id: Schema.String }), + role: Schema.optionalKey(Schema.String), +}) +export type ClerkOrganizationMembershipData = Schema.Schema.Type< + typeof ClerkOrganizationMembershipData +> + +export const decodeClerkOrganizationMembership = Schema.decodeUnknownEffect( + ClerkOrganizationMembershipData, +) + +/** The membership verbs Maple audits, keyed by Clerk's event type. */ +export const CLERK_MEMBERSHIP_EVENTS = { + "organizationMembership.created": "added", + "organizationMembership.updated": "role_changed", + "organizationMembership.deleted": "removed", +} as const satisfies Record + +export type ClerkMembershipEventType = keyof typeof CLERK_MEMBERSHIP_EVENTS + +export const isClerkMembershipEvent = (type: string): type is ClerkMembershipEventType => + Object.hasOwn(CLERK_MEMBERSHIP_EVENTS, type) + export const ClerkWebhookEnvelope = Schema.Struct({ type: Schema.String, data: Schema.Unknown, diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index 998325d98..17462b4de 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -120,6 +120,10 @@ "max_batch_size": 25, "max_batch_timeout": 5, "max_retries": 5, + // Mirrors alchemy.run.ts: exhausted audit entries park here rather + // than being dropped. Keep `max_retries` in sync with + // AUDIT_EVENTS_MAX_RETRIES in audit-events-runtime.ts. + "dead_letter_queue": "maple-audit-events-dlq-local", }, ], }, diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx index 967e28e92..6c6aab5d0 100644 --- a/apps/web/src/components/settings/audit-log-section.tsx +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -82,6 +82,19 @@ function formatSourceTooltip(entry: V2AuditLogEntry): string | undefined { return lines.length > 0 ? lines.join("\n") : undefined } +/** + * Append the next page, dropping any entry already shown. The pinned `until` + * ceiling makes overlap rare, but a filter re-fetch or a refresh mid-scroll can + * still repeat one — and a duplicated React key corrupts the list either way. + */ +function dedupeById( + existing: ReadonlyArray, + next: ReadonlyArray, +): V2AuditLogEntry[] { + const seen = new Set(existing.map((entry) => entry.id)) + return [...existing, ...next.filter((entry) => !seen.has(entry.id))] +} + interface AuditLogView { source: { data: ReadonlyArray } entries: V2AuditLogEntry[] @@ -93,9 +106,14 @@ export function AuditLogSection() { const [actorFilter, setActorFilter] = useState("all") const [outcomeFilter, setOutcomeFilter] = useState("all") const [cursor, setCursor] = useState(undefined) + // Frozen on the first Load more, and cleared whenever the list restarts. The + // log is append-only and paginated by offset, so entries written mid-scroll + // would otherwise shift later pages and make them repeat and skip rows. + const [until, setUntil] = useState(undefined) const pageAtom = auditLogPageAtom({ ...(cursor !== undefined ? { cursor } : undefined), + ...(until !== undefined ? { until } : undefined), ...(actorFilter !== "all" ? { actorType: actorFilter } : undefined), ...(outcomeFilter !== "all" ? { outcome: outcomeFilter } : undefined), }) @@ -112,7 +130,7 @@ export function AuditLogSection() { entries: cursor === undefined ? [...pageResult.value.data] - : [...(view?.entries ?? []), ...pageResult.value.data], + : dedupeById(view?.entries ?? [], pageResult.value.data), hasMore: pageResult.value.has_more, nextCursor: pageResult.value.next_cursor, }) @@ -122,12 +140,14 @@ export function AuditLogSection() { if (value === actorFilter) return setActorFilter(value) setCursor(undefined) + setUntil(undefined) } function handleOutcomeSelect(value: OutcomeFilter) { if (value === outcomeFilter) return setOutcomeFilter(value) setCursor(undefined) + setUntil(undefined) } const waiting = !Result.isSuccess(pageResult) || pageResult.waiting @@ -158,8 +178,10 @@ export function AuditLogSection() { ))}
+ {/* Deliberately not "every change": this records configuration and + access changes plus refused attempts, not reads or telemetry. */}

- Every change made through the dashboard, API, and MCP. + Configuration and access changes, from the dashboard, API, and MCP.

@@ -221,7 +243,15 @@ export function AuditLogSection() { size="sm" disabled={waiting} onClick={() => { - if (view.nextCursor !== null) setCursor(view.nextCursor) + if (view.nextCursor === null) return + // Pin the window to the newest entry already on screen before + // the first Load more, so later offsets address a list that + // cannot grow underneath them. + if (until === undefined) { + const newest = view.entries[0] + if (newest !== undefined) setUntil(newest.occurred_at) + } + setCursor(view.nextCursor) }} > {waiting ? "Loading…" : "Load more"} diff --git a/apps/web/src/lib/services/atoms/audit-log-atoms.ts b/apps/web/src/lib/services/atoms/audit-log-atoms.ts index 72772d2cf..1e1ccf353 100644 --- a/apps/web/src/lib/services/atoms/audit-log-atoms.ts +++ b/apps/web/src/lib/services/atoms/audit-log-atoms.ts @@ -12,17 +12,21 @@ export interface AuditLogPageInput { readonly cursor?: string readonly actorType?: AuditActorType readonly outcome?: AuditOutcome + /** + * Upper bound on `occurred_at`, pinned by the caller when it takes the first + * page. Pagination here is offset-based over a newest-first, append-only + * table, so an entry written mid-scroll shifts every later row down by one: + * without a frozen ceiling the next page repeats a row and skips another. + */ + readonly until?: string } -// Actor types and outcomes never contain "|", and the cursor is the trailing -// segment, so splitting on the first two separators stays unambiguous even for -// exotic cursors. +// Actor types and outcomes never contain "|", nor does an ISO timestamp, and the +// cursor is the trailing segment — so splitting on the first three separators +// stays unambiguous even for exotic cursors. const family = Atom.family((key: string) => { - const firstSeparator = key.indexOf("|") - const secondSeparator = key.indexOf("|", firstSeparator + 1) - const actorRaw = key.slice(0, firstSeparator) - const outcomeRaw = key.slice(firstSeparator + 1, secondSeparator) - const cursor = key.slice(secondSeparator + 1) + const [actorRaw = "", outcomeRaw = "", until = ""] = key.split("|", 3) + const cursor = key.slice(actorRaw.length + outcomeRaw.length + until.length + 3) const actorType = ACTOR_TYPES.find((type) => type === actorRaw) const outcome = OUTCOMES.find((value) => value === outcomeRaw) @@ -35,12 +39,13 @@ const family = Atom.family((key: string) => { ...(cursor !== "" ? { cursor } : undefined), ...(actorType !== undefined ? { actor_type: actorType } : undefined), ...(outcome !== undefined ? { outcome } : undefined), + ...(until !== "" ? { until } : undefined), }, }) }), ) }) -/** One page of the org's audit log, keyed by cursor + actor-type/outcome filters. */ +/** One page of the org's audit log, keyed by cursor + filters + the pinned ceiling. */ export const auditLogPageAtom = (input: AuditLogPageInput) => - family(`${input.actorType ?? ""}|${input.outcome ?? ""}|${input.cursor ?? ""}`) + family(`${input.actorType ?? ""}|${input.outcome ?? ""}|${input.until ?? ""}|${input.cursor ?? ""}`) From 8911c52b090548b7c8980c7db9e1dae25cc5f21c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 2 Sep 2026 15:36:32 +0200 Subject: [PATCH 07/19] wip(audit): ClickHouse-backed audit log + read auditing (pre-merge checkpoint) --- apps/api/src/services/audit/audit-actions.ts | 17 +++ .../services/warehouse/warehouse-catalog.ts | 13 ++- apps/cli/src/server/schema/local-inserts.json | 2 +- apps/cli/src/server/schema/local-schema.sql | 33 +++++- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- .../clickhouse/migrations/0025_audit_log.ts | 50 ++++++++ .../domain/src/generated/clickhouse-schema.ts | 3 +- .../generated/tinybird-project-manifest.ts | 7 +- packages/domain/src/http/audit-log.ts | 26 ++++- packages/domain/src/http/errors.ts | 11 +- packages/domain/src/http/query-engine.ts | 5 +- packages/domain/src/http/session-replay.ts | 7 +- packages/domain/src/http/v2/error-issues.ts | 2 + .../domain/src/http/v2/session-replays.ts | 2 + packages/domain/src/http/v2/telemetry.ts | 4 + packages/domain/src/tinybird/datasources.ts | 56 +++++++++ .../src/tinybird/retention-matrix.test.ts | 2 + .../query-engine/src/ch/builder-fixtures.ts | 50 ++++++++ packages/query-engine/src/ch/index.ts | 7 ++ .../query-engine/src/ch/queries/audit-log.ts | 110 ++++++++++++++++++ packages/query-engine/src/ch/tables.ts | 25 ++++ .../src/execution/datasource-routing.ts | 2 +- packages/query-engine/src/sql-catalog.test.ts | 2 + 23 files changed, 421 insertions(+), 17 deletions(-) create mode 100644 packages/domain/src/clickhouse/migrations/0025_audit_log.ts create mode 100644 packages/query-engine/src/ch/queries/audit-log.ts diff --git a/apps/api/src/services/audit/audit-actions.ts b/apps/api/src/services/audit/audit-actions.ts index 444284d52..2a6761421 100644 --- a/apps/api/src/services/audit/audit-actions.ts +++ b/apps/api/src/services/audit/audit-actions.ts @@ -40,6 +40,12 @@ export const AuditResources = { error_issue: { prefix: PublicIdPrefixes.errorIssue, verbs: ErrorIssueEventType.literals }, /** Org-singleton public/private pair; which one rolled is in `metadata`. */ ingest_key: { verbs: ["rolled"] }, + /** + * Every MCP tool invocation, whichever surface drove it (MCP transport, the + * in-app chat, workflows, internal RPC). The tool and its parameters are in + * `metadata`; a tool that also mutates a resource records that action too. + */ + mcp_tool: { verbs: ["called"] }, investigation: { prefix: PublicIdPrefixes.investigation, verbs: ["created", "restarted", "status_changed"] }, /** * Org-singleton connections. `*_started` is the admin action Maple sees; the @@ -61,6 +67,17 @@ export const AuditResources = { */ organization: { verbs: ["deleted"] }, scrape_target: { prefix: PublicIdPrefixes.scrapeTarget, verbs: ["created", "updated", "deleted"] }, + /** + * Reads of recorded browser sessions — the surface most likely to carry + * end-user data. Recorded by the auth layers from the `AuditedRead` annotation. + */ + session_replay: { verbs: ["read"] }, + /** + * Reads of traces, logs, metrics and error events (`read`, from the + * `AuditedRead` annotation on the endpoint) and every raw SQL statement run + * against the warehouse (`sql_executed`, with the statement in `metadata`). + */ + telemetry: { verbs: ["read", "sql_executed"] }, /** Org-singleton BYO-ClickHouse connection; holds warehouse credentials. */ warehouse_settings: { verbs: ["updated", "deleted", "schema_applied"] }, /** Short-lived device credentials for the mobile widget; keyed by installation. */ diff --git a/apps/api/src/services/warehouse/warehouse-catalog.ts b/apps/api/src/services/warehouse/warehouse-catalog.ts index 7f48c8d28..f3f8f6af8 100644 --- a/apps/api/src/services/warehouse/warehouse-catalog.ts +++ b/apps/api/src/services/warehouse/warehouse-catalog.ts @@ -91,12 +91,23 @@ export interface TableInfo extends TableSummary { readonly partitionKey?: string } +/** + * Datasources that raw SQL must never reach, even inside the caller's own org. + * The audit log records every member's activity and origin IP and is served + * only through the admin-gated `GET /v2/audit_log`; letting `run_sql` or a + * dashboard widget read it would bypass that gate (and let the log observe + * itself being read). + */ +const RAW_SQL_HIDDEN_DATASOURCES: ReadonlySet = new Set(["audit_log"]) + function collectDatasources() { // `Datasources` exports a mix of datasource definitions, type aliases, helper // functions, and constant lookup tables. `isDatasourceDefinition` is the // runtime filter; we cast to `unknown` first because the static union of all // exports is too wide for TS to narrow with the predicate. - return (Object.values(Datasources) as ReadonlyArray).filter(isDatasourceDefinition) + return (Object.values(Datasources) as ReadonlyArray) + .filter(isDatasourceDefinition) + .filter((ds) => !RAW_SQL_HIDDEN_DATASOURCES.has(ds._name)) } export function listWarehouseTables(): ReadonlyArray { diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 6b227c7d3..31467961a 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "f27af9955a90e5deb9f1351c967a749f0a2db0ceebb85b4d0f669fc4c575ee35", + "projectRevision": "57f40b9184ff2799b32ce295dab3309a8b44460f419fdf0eff09c90b97d310cc", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index ade8d74f6..2f2cec114 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: f27af9955a90e5deb9f1351c967a749f0a2db0ceebb85b4d0f669fc4c575ee35 --- localSchemaVersion: 11 +-- projectRevision: 57f40b9184ff2799b32ce295dab3309a8b44460f419fdf0eff09c90b97d310cc +-- localSchemaVersion: 12 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -55,6 +55,35 @@ PARTITION BY toDate(Hour) ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) TTL Hour + INTERVAL 90 DAY; +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + CREATE TABLE IF NOT EXISTS error_events ( OrgId LowCardinality(String), Timestamp DateTime, diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index f98f5067c..e915f6bc0 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "f27af9955a90e5deb9f1351c967a749f0a2db0ceebb85b4d0f669fc4c575ee35"; +pub const PROJECT_REVISION: &str = "57f40b9184ff2799b32ce295dab3309a8b44460f419fdf0eff09c90b97d310cc"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/clickhouse/migrations/0025_audit_log.ts b/packages/domain/src/clickhouse/migrations/0025_audit_log.ts new file mode 100644 index 000000000..261066cb3 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0025_audit_log.ts @@ -0,0 +1,50 @@ +/** + * 0025 — `audit_log`: the org-wide audit trail, moved out of Postgres. + * + * Written only by the API worker through the managed Tinybird pipeline and read + * only by the admin-gated `GET /v2/audit_log`. It ships in the migration set so + * self-hosted deployments (where the "managed" pipeline IS this ClickHouse) have + * the table; a BYO-ClickHouse org never reads or writes it — reads are pinned to + * the managed route (`INGEST_PINNED_TABLES`). + * + * `requiredForIngest: false`: the ingest gateway writes nothing here, so the + * table's presence must not gate an org's ingest readiness. + * + * Retention is six years (HIPAA §164.316(b)(2)); `''` stands in for absent + * values throughout — see the datasource definition for the column contract. + */ +export const migration_0025_audit_log = { + version: 25, + description: "Create audit_log, the org-wide audit trail (actions, denials, and telemetry reads).", + requiredForIngest: false, + statements: [ + `CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY`, + ], +} as const diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index e5058a8d8..ac008b706 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,12 +1,13 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "f27af9955a90e5deb9f1351c967a749f0a2db0ceebb85b4d0f669fc4c575ee35" as const +export const projectRevision = "57f40b9184ff2799b32ce295dab3309a8b44460f419fdf0eff09c90b97d310cc" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS attribute_keys_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, Hour, AttributeKey)\nTTL Hour + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS attribute_values_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue)\nTTL Hour + INTERVAL 90 DAY", + "CREATE TABLE IF NOT EXISTS audit_log (\n OrgId LowCardinality(String),\n Id String,\n OccurredAt DateTime64(3),\n RecordedAt DateTime64(3),\n ActorType LowCardinality(String),\n UserId String,\n ApiKeyId String,\n ActorId String,\n ActorLabel String,\n AffectedUserId String,\n Source LowCardinality(String),\n Action LowCardinality(String),\n Outcome LowCardinality(String),\n DenialReason String,\n ResourceType LowCardinality(String),\n ResourceId String,\n ChangedFields Array(String),\n Changes String,\n Metadata String,\n RequestId String,\n OriginIp String,\n OriginCountry LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(OccurredAt)\nORDER BY (OrgId, OccurredAt, Id)\nTTL toDate(OccurredAt) + INTERVAL 2190 DAY", "CREATE TABLE IF NOT EXISTS error_events (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, FingerprintHash, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String,\n ServiceVersion LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_fingerprints_minutely (\n OrgId LowCardinality(String),\n Minute DateTime,\n FingerprintHash UInt64,\n ServiceName SimpleAggregateFunction(anyLast, String),\n ExceptionType SimpleAggregateFunction(anyLast, String),\n ExceptionMessage SimpleAggregateFunction(anyLast, String),\n ErrorLabel SimpleAggregateFunction(anyLast, String),\n TopFrame SimpleAggregateFunction(anyLast, String),\n OccurrenceCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime),\n ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toYYYYMM(Minute)\nORDER BY (OrgId, Minute, FingerprintHash)\nTTL Minute + INTERVAL 90 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 2f5274594..3001f6316 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "f27af9955a90e5deb9f1351c967a749f0a2db0ceebb85b4d0f669fc4c575ee35" as const +export const projectRevision = "57f40b9184ff2799b32ce295dab3309a8b44460f419fdf0eff09c90b97d310cc" as const export const datasources = [ { @@ -19,6 +19,11 @@ export const datasources = [ content: 'DESCRIPTION >\n Pre-aggregated attribute values with hourly usage counts from trace span and resource attributes.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, AttributeScope, AttributeKey, Hour, AttributeValue"\nENGINE_TTL "Hour + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT *', }, + { + name: "audit_log", + content: + 'DESCRIPTION >\n Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n RecordedAt DateTime64(3) `json:$.RecordedAt`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n UserId String `json:$.UserId`,\n ApiKeyId String `json:$.ApiKeyId`,\n ActorId String `json:$.ActorId`,\n ActorLabel String `json:$.ActorLabel`,\n AffectedUserId String `json:$.AffectedUserId`,\n Source LowCardinality(String) `json:$.Source`,\n Action LowCardinality(String) `json:$.Action`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n DenialReason String `json:$.DenialReason`,\n ResourceType LowCardinality(String) `json:$.ResourceType`,\n ResourceId String `json:$.ResourceId`,\n ChangedFields Array(String) `json:$.ChangedFields`,\n Changes String `json:$.Changes`,\n Metadata String `json:$.Metadata`,\n RequestId String `json:$.RequestId`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 2190 DAY"', + }, { name: "error_events", content: diff --git a/packages/domain/src/http/audit-log.ts b/packages/domain/src/http/audit-log.ts index bafd703a8..6ed98fada 100644 --- a/packages/domain/src/http/audit-log.ts +++ b/packages/domain/src/http/audit-log.ts @@ -1,4 +1,4 @@ -import { Schema } from "effect" +import { Context, Schema } from "effect" import { HttpTaggedError } from "./error-policy" /** @@ -37,6 +37,30 @@ export const AuditChanges = Schema.Struct({ }) export type AuditChanges = Schema.Schema.Type +/** + * The audit action a data-read endpoint records. Telemetry (traces, logs, + * metrics, error events) and session replays are the two surfaces that can + * carry customer end-user data, so every read of them is logged — HIPAA audit + * controls cover access, not only change. + */ +export const AuditReadAction = Schema.Literals(["telemetry.read", "session_replay.read"]).annotate({ + identifier: "@maple/AuditReadAction", + title: "Audit Read Action", +}) +export type AuditReadAction = Schema.Schema.Type + +/** + * Endpoint/group annotation declaring that a successful call is a data read + * worth an audit entry. The auth middlewares consult it on every request; an + * endpoint without it (configuration, billing, the audit log itself) records + * nothing on reads. Declared here, next to the contracts, so "which endpoints + * expose telemetry" is visible where the endpoints are. + */ +export class AuditedRead extends Context.Reference( + "@maple/http/AuditedRead", + { defaultValue: () => undefined }, +) {} + export class AuditLogPersistenceError extends HttpTaggedError()( "@maple/http/errors/AuditLogPersistenceError", { diff --git a/packages/domain/src/http/errors.ts b/packages/domain/src/http/errors.ts index 89db688fd..ff19c7d84 100644 --- a/packages/domain/src/http/errors.ts +++ b/packages/domain/src/http/errors.ts @@ -16,6 +16,7 @@ import { TraceId, UserId, } from "../primitives" +import { AuditedRead } from "./audit-log" import { Authorization } from "./current-tenant" import { AlertSeverity } from "./alerts" import { @@ -881,7 +882,7 @@ export class ErrorsApiGroup extends HttpApiGroup.make("errors") query: IssueListQuery, success: ErrorIssuesListResponse, error: ErrorPersistenceError, - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.get("getIssue", "/issues/:issueId", { @@ -889,7 +890,7 @@ export class ErrorsApiGroup extends HttpApiGroup.make("errors") query: IssueDetailQuery, success: ErrorIssueDetailResponse, error: [ErrorPersistenceError, ErrorIssueNotFoundError], - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.post("transitionIssue", "/issues/:issueId/transitions", { @@ -985,20 +986,20 @@ export class ErrorsApiGroup extends HttpApiGroup.make("errors") query: IssueEventsQuery, success: ErrorIssueEventsResponse, error: [ErrorPersistenceError, ErrorIssueNotFoundError], - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.get("listIssueIncidents", "/issues/:issueId/incidents", { params: { issueId: ErrorIssueId }, success: ErrorIncidentsListResponse, error: [ErrorPersistenceError, ErrorIssueNotFoundError], - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.get("listOpenIncidents", "/incidents", { success: ErrorIncidentsListResponse, error: ErrorPersistenceError, - }), + }).annotate(AuditedRead, "telemetry.read"), ) .add( HttpApiEndpoint.post("registerAgent", "/agents", { diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index ed0036b29..f9b2f3eb1 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -20,6 +20,7 @@ import { QueryEngineExecuteResponse, TinybirdDateTime, } from "../query-engine" +import { AuditedRead } from "./audit-log" import { SessionAuthorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { warehouseHttpErrors } from "./warehouse" @@ -2430,4 +2431,6 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") }), ) .prefix("/internal/query-engine") - .middleware(SessionAuthorization) {} + .middleware(SessionAuthorization) + // Every endpoint here reads telemetry for the dashboard. + .annotate(AuditedRead, "telemetry.read") {} diff --git a/packages/domain/src/http/session-replay.ts b/packages/domain/src/http/session-replay.ts index b2583ef05..749c04219 100644 --- a/packages/domain/src/http/session-replay.ts +++ b/packages/domain/src/http/session-replay.ts @@ -2,6 +2,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" import { Schema } from "effect" import { SessionId, TraceId, UserId } from "../primitives" import { TinybirdDateTime } from "../query-engine" +import { AuditedRead } from "./audit-log" import { Authorization, SessionAuthorization } from "./current-tenant" import { QueryEngineExecutionError, QueryEngineTimeoutError } from "./query-engine" import { warehouseHttpErrors } from "./warehouse" @@ -330,7 +331,8 @@ export class SessionReplaysApiGroup extends HttpApiGroup.make("sessionReplays") }), ) .prefix("/api/session-replays") - .middleware(Authorization) {} + .middleware(Authorization) + .annotate(AuditedRead, "session_replay.read") {} /** * Session-replay helpers that exist for the dashboard and are not public API. @@ -356,4 +358,5 @@ export class SessionReplaysInternalApiGroup extends HttpApiGroup.make("sessionRe }), ) .prefix("/internal/session-replays") - .middleware(SessionAuthorization) {} + .middleware(SessionAuthorization) + .annotate(AuditedRead, "session_replay.read") {} diff --git a/packages/domain/src/http/v2/error-issues.ts b/packages/domain/src/http/v2/error-issues.ts index f56696499..ec48fd0f1 100644 --- a/packages/domain/src/http/v2/error-issues.ts +++ b/packages/domain/src/http/v2/error-issues.ts @@ -10,6 +10,7 @@ import { WorkflowState, } from "../errors" import { SpanId, TraceId, UserId } from "../../primitives" +import { AuditedRead } from "../audit-log" import { AuthorizationV2 } from "./auth" import { ListOf, ListQuery, Timestamp } from "./envelopes" import { V2CursorInvalid, V2CursorSortMismatch } from "./errors" @@ -242,6 +243,7 @@ export class V2ErrorIssuesApiGroup extends HttpApiGroup.make("errorIssues") ) .prefix("/v2/error_issues") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Error Issues", diff --git a/packages/domain/src/http/v2/session-replays.ts b/packages/domain/src/http/v2/session-replays.ts index cad31dfeb..1ad0f1354 100644 --- a/packages/domain/src/http/v2/session-replays.ts +++ b/packages/domain/src/http/v2/session-replays.ts @@ -1,6 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" import { Schema } from "effect" import { SessionId, TraceId } from "../../primitives" +import { AuditedRead } from "../audit-log" import { AuthorizationV2 } from "./auth" import { wireExample, ListOf, ListQuery, Timestamp } from "./envelopes" import { defineV2Error, V2ParameterInvalid } from "./errors" @@ -578,6 +579,7 @@ export class V2SessionReplaysApiGroup extends HttpApiGroup.make("sessionReplays" ) .prefix("/v2/session_replays") .middleware(AuthorizationV2) + .annotate(AuditedRead, "session_replay.read") .annotateMerge( OpenApi.annotations({ title: "Session Replays", diff --git a/packages/domain/src/http/v2/telemetry.ts b/packages/domain/src/http/v2/telemetry.ts index 80b37e0b4..6732ade05 100644 --- a/packages/domain/src/http/v2/telemetry.ts +++ b/packages/domain/src/http/v2/telemetry.ts @@ -1,6 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" import { Schema } from "effect" import { MetricName, ServiceName, SpanId, TraceId } from "../../primitives" +import { AuditedRead } from "../audit-log" import { AuthorizationV2 } from "./auth" import { wireExample, ListOf, ListQuery, Timestamp } from "./envelopes" import { defineV2Error, V2CursorInvalid, V2ParameterInvalid, V2TimeRangeInvalid } from "./errors" @@ -765,6 +766,7 @@ export class V2TracesApiGroup extends HttpApiGroup.make("traces") ) .prefix("/v2/traces") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Traces", @@ -867,6 +869,7 @@ export class V2LogsApiGroup extends HttpApiGroup.make("logs") ) .prefix("/v2/logs") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Logs", @@ -947,6 +950,7 @@ export class V2MetricsApiGroup extends HttpApiGroup.make("metrics") ) .prefix("/v2/metrics") .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") .annotateMerge( OpenApi.annotations({ title: "Metrics", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 0ca0de572..0341ce7ad 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1496,6 +1496,62 @@ export const alertChecks = defineDatasource("alert_checks", { export type AlertChecksRow = InferRow +/** + * The org-wide audit log: one row per allowed or denied action, and per read + * of telemetry or session replays, attributed to the user, API key, agent, or + * Maple itself that performed it. Written by the API through `ingest` (the + * audit events queue consumer, or the producer directly when no queue is + * bound) and read only by the admin-gated `GET /v2/audit_log`; never fed by a + * materialized view and never routed to a BYO ClickHouse — the log is Maple's + * record, not the customer warehouse's. + * + * Absent values are empty strings rather than NULL: `LowCardinality(Nullable)` + * is awkward in ClickHouse and every read maps `''` back to `null` on the wire. + * `Changes`/`Metadata` hold JSON documents (`''` when none); `ChangedFields` + * keeps the touched field names queryable without parsing `Changes`. + * + * Six-year retention: HIPAA §164.316(b)(2) keeps required documentation for six + * years, and the audit trail is the documentation of who accessed what. + */ +export const auditLog = defineDatasource("audit_log", { + description: + "Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.", + schema: { + OrgId: t.string().lowCardinality(), + Id: t.string(), + OccurredAt: t.dateTime64(3), + RecordedAt: t.dateTime64(3), + ActorType: t.string().lowCardinality(), + UserId: t.string(), + ApiKeyId: t.string(), + ActorId: t.string(), + ActorLabel: t.string(), + AffectedUserId: t.string(), + Source: t.string().lowCardinality(), + Action: t.string().lowCardinality(), + Outcome: t.string().lowCardinality(), + DenialReason: t.string(), + ResourceType: t.string().lowCardinality(), + ResourceId: t.string(), + ChangedFields: t.array(t.string()), + Changes: t.string(), + Metadata: t.string(), + RequestId: t.string(), + OriginIp: t.string(), + OriginCountry: t.string().lowCardinality(), + }, + // ReplacingMergeTree keyed on the entry id makes queue redelivery idempotent: + // a second delivery of the same event collapses at merge time instead of + // showing as a duplicate row. Reads dedupe the (rare) pre-merge window too. + engine: engine.replacingMergeTree({ + partitionKey: "toYYYYMM(OccurredAt)", + sortingKey: ["OrgId", "OccurredAt", "Id"], + ttl: "toDate(OccurredAt) + INTERVAL 2190 DAY", + }), +}) + +export type AuditLogRow = InferRow + /** * Minute-grain operation metrics used by the service-detail Operations panel. * The operation name is normalized once by the write-side MV, while exact and diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index c3c328e52..0b7b00147 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -3,6 +3,8 @@ import { tinybirdProjectManifest } from "../generated/tinybird-project-manifest" const RETENTION_DAYS = { alert_checks: 365, + // Six years — HIPAA's documentation retention floor. Never rebuildable. + audit_log: 2190, attribute_keys_hourly: 90, attribute_values_hourly: 90, error_events: 90, diff --git a/packages/query-engine/src/ch/builder-fixtures.ts b/packages/query-engine/src/ch/builder-fixtures.ts index d20ace8eb..53ce13785 100644 --- a/packages/query-engine/src/ch/builder-fixtures.ts +++ b/packages/query-engine/src/ch/builder-fixtures.ts @@ -315,6 +315,56 @@ const productEventsFixtures: ReadonlyArray = [ export const builderFixtures: ReadonlyArray = [ ...productEventsFixtures, + // Audit log listing (apps/api/src/services/audit/AuditLogService.ts `list`). + { + module: "audit-log", + name: "auditLogEntriesQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.auditLogEntriesQuery({ limit: 50, offset: 0 }), { orgId: ORG_ID }), + }, + { + // Every optional filter bound at once, including the raw `has(...)` clause. + module: "audit-log", + name: "auditLogEntriesQuery", + label: "filtered", + compile: () => + CH.compileUnsafe( + CH.auditLogEntriesQuery({ + actorType: true, + userId: true, + apiKeyId: true, + actorId: true, + affectedUserId: true, + action: true, + outcome: true, + resourceType: true, + resourceId: true, + changedField: true, + requestId: true, + since: true, + until: true, + limit: 50, + offset: 50, + }), + { + orgId: ORG_ID, + actorType: "user", + userId: "user_1", + apiKeyId: "key_1", + actorId: "actor_1", + affectedUserId: "user_2", + action: "dashboard.updated", + outcome: "allowed", + resourceType: "dashboard", + resourceId: "dash_1", + changedField: "name", + requestId: "ray", + since: START_TIME, + until: END_TIME, + }, + ), + }, // Session replay fixtures used by the replay routes. { module: "session-replays", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index dc295ce88..fcb61257e 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -400,6 +400,13 @@ export { type AlertChecksSummaryOutput, } from "./queries/alert-checks" +// Queries — Audit log (org-wide audit trail, admin-only) +export { + auditLogEntriesQuery, + type AuditLogEntriesOpts, + type AuditLogEntriesOutput, +} from "./queries/audit-log" + // Queries — Cloudflare integration usage (integrations-page ingest proof) // Queries — Cloudflare service-map stats (per-zone / per-Worker node rollups) diff --git a/packages/query-engine/src/ch/queries/audit-log.ts b/packages/query-engine/src/ch/queries/audit-log.ts new file mode 100644 index 000000000..e08ddc9ef --- /dev/null +++ b/packages/query-engine/src/ch/queries/audit-log.ts @@ -0,0 +1,110 @@ +import * as CH from "@maple-dev/clickhouse-builder/expr" +import { from, param, paramPlaceholder } from "@maple-dev/clickhouse-builder" +import { AuditLog } from "../tables" + +/** + * Which optional filters a listing applies. Every set flag binds a parameter of + * the same name at compile time; the values themselves never enter the SQL. + */ +export interface AuditLogEntriesOpts { + readonly actorType?: boolean + readonly userId?: boolean + readonly apiKeyId?: boolean + readonly actorId?: boolean + readonly affectedUserId?: boolean + readonly action?: boolean + readonly outcome?: boolean + readonly resourceType?: boolean + readonly resourceId?: boolean + readonly changedField?: boolean + readonly requestId?: boolean + readonly since?: boolean + readonly until?: boolean + readonly limit: number + readonly offset: number +} + +/** + * One org's audit log, newest first, offset-paginated. + * + * `LIMIT … BY Id` collapses a redelivered entry that ReplacingMergeTree has not + * merged yet, so a page never shows the same entry twice. Pinned to the managed + * route: the table is written through `ingest` and does not exist in a BYO + * ClickHouse. + */ +export function auditLogEntriesQuery(opts: AuditLogEntriesOpts) { + return from(AuditLog) + .select(($) => ({ + id: $.Id, + occurredAt: $.OccurredAt, + recordedAt: $.RecordedAt, + actorType: $.ActorType, + userId: $.UserId, + apiKeyId: $.ApiKeyId, + actorId: $.ActorId, + actorLabel: $.ActorLabel, + affectedUserId: $.AffectedUserId, + source: $.Source, + action: $.Action, + outcome: $.Outcome, + denialReason: $.DenialReason, + resourceType: $.ResourceType, + resourceId: $.ResourceId, + changedFields: $.ChangedFields, + changes: $.Changes, + metadata: $.Metadata, + requestId: $.RequestId, + originIp: $.OriginIp, + originCountry: $.OriginCountry, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + opts.actorType ? $.ActorType.eq(param.string("actorType")) : undefined, + opts.userId ? $.UserId.eq(param.string("userId")) : undefined, + opts.apiKeyId ? $.ApiKeyId.eq(param.string("apiKeyId")) : undefined, + opts.actorId ? $.ActorId.eq(param.string("actorId")) : undefined, + opts.affectedUserId ? $.AffectedUserId.eq(param.string("affectedUserId")) : undefined, + opts.action ? $.Action.eq(param.string("action")) : undefined, + opts.outcome ? $.Outcome.eq(param.string("outcome")) : undefined, + opts.resourceType ? $.ResourceType.eq(param.string("resourceType")) : undefined, + opts.resourceId ? $.ResourceId.eq(param.string("resourceId")) : undefined, + // Array membership has no builder verb yet; the placeholder keeps the + // value parameterised exactly like the typed comparisons above. + opts.changedField + ? CH.rawCond(`has(ChangedFields, ${paramPlaceholder("string", "changedField")})`) + : undefined, + opts.requestId ? $.RequestId.eq(param.string("requestId")) : undefined, + opts.since ? $.OccurredAt.gte(param.dateTimeString("since")) : undefined, + opts.until ? $.OccurredAt.lte(param.dateTimeString("until")) : undefined, + ]) + .orderBy(["occurredAt", "desc"], ["id", "desc"]) + .limit(opts.limit) + .offset(opts.offset) + .format("JSON") + .route("ingest") +} + +/** One listed entry as the warehouse returns it: `''` for absent values, JSON text for documents. */ +export interface AuditLogEntriesOutput { + readonly id: string + readonly occurredAt: string + readonly recordedAt: string + readonly actorType: string + readonly userId: string + readonly apiKeyId: string + readonly actorId: string + readonly actorLabel: string + readonly affectedUserId: string + readonly source: string + readonly action: string + readonly outcome: string + readonly denialReason: string + readonly resourceType: string + readonly resourceId: string + readonly changedFields: ReadonlyArray + readonly changes: string + readonly metadata: string + readonly requestId: string + readonly originIp: string + readonly originCountry: string +} diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 7339a7b91..dd04a1f4d 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -581,6 +581,31 @@ export const AlertChecks = table("alert_checks", { ErrorCategory: T.string, }) +export const AuditLog = table("audit_log", { + OrgId: orgId, + Id: T.string, + OccurredAt: dateTime64, + RecordedAt: dateTime64, + ActorType: T.string, + UserId: T.string, + ApiKeyId: T.string, + ActorId: T.string, + ActorLabel: T.string, + AffectedUserId: T.string, + Source: T.string, + Action: T.string, + Outcome: T.string, + DenialReason: T.string, + ResourceType: T.string, + ResourceId: T.string, + ChangedFields: T.array(T.string), + Changes: T.string, + Metadata: T.string, + RequestId: T.string, + OriginIp: T.string, + OriginCountry: T.string, +}) + export const SessionReplays = table("session_replays", { OrgId: orgId, SessionId: T.string, diff --git a/packages/query-engine/src/execution/datasource-routing.ts b/packages/query-engine/src/execution/datasource-routing.ts index a23cfc091..9a42bca5d 100644 --- a/packages/query-engine/src/execution/datasource-routing.ts +++ b/packages/query-engine/src/execution/datasource-routing.ts @@ -8,7 +8,7 @@ * org-BYO backend while referencing one of these tables would silently return * empty rows, so it logs a warning instead of failing quietly. */ -export const INGEST_PINNED_TABLES: ReadonlyArray = ["alert_checks"] +export const INGEST_PINNED_TABLES: ReadonlyArray = ["alert_checks", "audit_log"] export const findIngestPinnedTable = (sql: string): string | undefined => INGEST_PINNED_TABLES.find((table) => sql.includes(table)) diff --git a/packages/query-engine/src/sql-catalog.test.ts b/packages/query-engine/src/sql-catalog.test.ts index a6341f4ca..72563cdf8 100644 --- a/packages/query-engine/src/sql-catalog.test.ts +++ b/packages/query-engine/src/sql-catalog.test.ts @@ -19,6 +19,7 @@ import { import { builderFixtures } from "./ch/builder-fixtures" import * as activityQueries from "./ch/queries/activity" import * as alertCheckQueries from "./ch/queries/alert-checks" +import * as auditLogQueries from "./ch/queries/audit-log" import * as anomalyQueries from "./ch/queries/anomaly" import * as attributeKeyQueries from "./ch/queries/attribute-keys" import * as errorQueries from "./ch/queries/errors" @@ -216,6 +217,7 @@ describe("sql catalog", () => { const QUERY_MODULES: Record> = { activity: activityQueries, "alert-checks": alertCheckQueries, + "audit-log": auditLogQueries, anomaly: anomalyQueries, "attribute-keys": attributeKeyQueries, errors: errorQueries, From 8629094f27b410788047a5def64e4ce14c3ec882 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 2 Sep 2026 16:00:46 +0200 Subject: [PATCH 08/19] feat(audit): store the audit log in ClickHouse and record data reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HIPAA audit controls cover access, not only change, and an audit trail that grows with every telemetry read belongs in the warehouse, not in the application database. Storage moves from Postgres (`audit_log_entries`, migration 0050, the hourly retention sweep and `AUDIT_LOG_RETENTION_DAYS` are removed) to the Tinybird datasource `audit_log` — ClickHouse migration 0025, ReplacingMergeTree on the entry id so queue redelivery collapses at merge, monthly partitions, six-year TTL (§164.316(b)(2)). Writes go through `WarehouseQueryService.ingest`, which is pinned to the managed pipeline and never a BYO ClickHouse; reads through the new `auditLogEntriesQuery` builder, routed the same way and listed on `INGEST_PINNED_TABLES`. The datasource is hidden from raw SQL and the per-org read JWT so the admin gate on `GET /v2/audit_log` is the only way in. The queue consumer builds the warehouse layer and ingests one batch per org; ack/retry/DLQ semantics are unchanged. Reads are now recorded on three surfaces: - HTTP endpoints annotated `AuditedRead` in the domain contracts — `telemetry.read` on the internal query-engine group, v2 traces/logs/ metrics/error_issues and the v1 error-issue GETs, `session_replay.read` on every replay group. The three auth layers wrap the handler with `withAuditedRead`, recording endpoint, method, path, status and a bounded body snapshot with the request's forensics. - Every MCP tool invocation from any surface (`mcp_tool.called`, with the tool and its parameters), from the executor. - Every raw SQL statement (`telemetry.sql_executed`) from `run_sql`, `inspect_chart_data` and the dashboard's raw-SQL route — a statement the safety pass refuses is recorded as `denied`. Tests use `AuditLogService.layerMemory`, an in-memory implementation with the query's filter and ordering semantics. The local chDB schema moves to v15 with an additive migration edge. --- apps/alerting/src/worker.ts | 6 +- apps/api/alchemy.run.ts | 2 - apps/api/src/audit-events-runtime.test.ts | 139 +- apps/api/src/audit-events-runtime.ts | 192 +- apps/api/src/mcp/dispatcher.test.ts | 4 +- apps/api/src/mcp/dispatcher.ts | 13 +- apps/api/src/mcp/lib/run-raw-sql.ts | 22 +- apps/api/src/platform/time.ts | 11 + .../src/routes/internal/query-engine.http.ts | 22 +- .../v2/alchemy-provider.integration.test.ts | 2 +- apps/api/src/routes/v2/alerts.http.test.ts | 2 +- apps/api/src/routes/v2/api-keys.http.test.ts | 2 +- apps/api/src/routes/v2/audit-log.http.ts | 17 +- .../routes/v2/config-resources.http.test.ts | 2 +- .../api/src/routes/v2/dashboards.http.test.ts | 2 +- .../src/routes/v2/integrations.http.test.ts | 2 +- .../src/routes/v2/mobile-devices.http.test.ts | 2 +- .../routes/v2/phase1-resources.http.test.ts | 2 +- .../src/routes/v2/setup-audit.http.test.ts | 2 +- apps/api/src/routes/v2/telemetry.http.test.ts | 2 +- apps/api/src/routes/v2/v2-test-support.ts | 4 +- .../routes/v2/widget-credentials.http.test.ts | 2 +- .../src/routes/v2/widget-summary.http.test.ts | 2 +- .../src/routes/webhooks/webhooks.http.test.ts | 3 +- apps/api/src/runtime/graph-boundaries.test.ts | 2 +- apps/api/src/runtime/http-graph.ts | 6 +- apps/api/src/runtime/mcp-service-graph.ts | 6 +- apps/api/src/runtime/service-graph.ts | 11 +- apps/api/src/services/alerts/AlertsService.ts | 10 +- .../services/audit/AuditLogService.test.ts | 353 ++- .../api/src/services/audit/AuditLogService.ts | 343 +-- apps/api/src/services/audit/audit-access.ts | 223 ++ apps/api/src/services/audit/audit-event.ts | 151 +- .../services/auth/ApiAuthorizationLayer.ts | 13 +- .../services/auth/ApiAuthorizationV2Layer.ts | 14 +- .../auth/SessionAuthorizationLayer.ts | 14 +- .../ErrorIssueReadModelsService.test.ts | 2 +- .../errors/ErrorIssueWorkflowService.test.ts | 6 +- .../src/services/errors/ErrorsService.test.ts | 4 +- .../IssueFixVerificationService.test.ts | 2 +- apps/api/src/vcs-sync-runtime.ts | 22 +- apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 2 + .../v14-to-v15-audit-log.ts | 150 ++ apps/cli/src/server/schema-identity.ts | 7 +- .../src/server/schema/local-schema-v15.sql | 1936 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 2 +- apps/cli/test/local-store-migrations.test.ts | 22 +- .../components/settings/audit-log-section.tsx | 7 +- .../src/clickhouse/migrations/index.test.ts | 10 +- 51 files changed, 3311 insertions(+), 475 deletions(-) create mode 100644 apps/api/src/services/audit/audit-access.ts create mode 100644 apps/cli/src/server/local-store-migrations/v14-to-v15-audit-log.ts create mode 100644 apps/cli/src/server/schema/local-schema-v15.sql diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index d4f1d03fb..27aaa7dad 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -143,7 +143,11 @@ export const buildLayer = (env: AlertingWorkerEnv) => { const ErrorActorsServiceLive = ErrorActorsService.layer.pipe(Layer.provide(BaseLive)) const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( Layer.provide( - Layer.mergeAll(BaseLive, ErrorActorsServiceLive, AuditLogService.layer.pipe(Layer.provide(BaseLive))), + Layer.mergeAll( + BaseLive, + ErrorActorsServiceLive, + AuditLogService.layer.pipe(Layer.provide(WarehouseQueryServiceLive)), + ), ), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer.pipe(Layer.provide(BaseLive)) diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 966428621..2c4048092 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -197,8 +197,6 @@ const apiConfiguredEnv = (stage: MapleStage, domains: MapleDomains) => // Agent LLM path. `MAPLE_LLM_PROVIDER` flips between OpenRouter (default) and // Workers AI; both stay wired, so a switch is this one var plus a redeploy. // See `@/platform/Llm` for the provider-scoped model overrides. - // Audit log retention horizon in days; the sweep defaults to 400 when unset. - optionalPlain("AUDIT_LOG_RETENTION_DAYS"), optionalPlain("MAPLE_LLM_PROVIDER"), optionalPlain("MAPLE_TRIAGE_MODEL_OPENROUTER"), optionalPlain("MAPLE_TRIAGE_MODEL_WORKERS_AI"), diff --git a/apps/api/src/audit-events-runtime.test.ts b/apps/api/src/audit-events-runtime.test.ts index 072c9456a..958617264 100644 --- a/apps/api/src/audit-events-runtime.test.ts +++ b/apps/api/src/audit-events-runtime.test.ts @@ -1,22 +1,21 @@ -import { afterEach, describe, expect, it } from "@effect/vitest" +import { describe, expect, it } from "@effect/vitest" import { OrgId } from "@maple/domain/primitives" +import type { AuditLogRow } from "@maple/domain/tinybird" +import { WarehouseUpstreamError } from "@maple/domain/http" import { Effect, Layer, Schema } from "effect" -import { auditLogEntries } from "@maple/db" -import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" -import { Database, DatabaseError } from "@/platform/DatabaseLive" import { processAuditEventsBatch } from "./audit-events-runtime" +import { makeWarehouseServiceStub } from "@/routes/v2/v2-test-support" import { AuditLogEvent, encodeAuditLogEventSync } from "./services/audit/audit-event" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" const asOrgId = Schema.decodeUnknownSync(OrgId) const ORG = asOrgId("org_audit_consumer_test") -const createdDbs: TestDb[] = [] +const OTHER_ORG = asOrgId("org_audit_consumer_other") -afterEach(() => cleanupTestDbs(createdDbs)) - -const event = (id: string) => +const event = (id: string, orgId: OrgId = ORG) => encodeAuditLogEventSync( new AuditLogEvent({ - orgId: ORG, + orgId, id: Schema.decodeUnknownSync(AuditLogEvent.fields.id)(id), actorType: "user", source: "dashboard", @@ -40,88 +39,84 @@ const message = (body: unknown, attempts: number) => { } } -const run = (effect: Effect.Effect) => { - const db = createTestDb(createdDbs) - return effect.pipe(Effect.provide(db.layer)) -} - const batchOf = (...messages: ReadonlyArray<{ readonly message: unknown }>) => ({ messages: messages.map((entry) => entry.message) }) as never -/** - * A database whose every write fails, so the consumer's retry path is exercised - * without depending on a real Postgres fault. - */ -const failingDatabase = Layer.succeed(Database, { - execute: () => - Effect.fail(new DatabaseError({ message: "insert failed", cause: new Error("insert failed") })), -}) - -describe("processAuditEventsBatch", () => { - it.effect("inserts a well-formed event and acks it", () => - run( - Effect.gen(function* () { - const first = message(event("11111111-1111-4111-8111-111111111111"), 1) - yield* processAuditEventsBatch(batchOf(first)) - - expect(first.calls).toEqual(["ack"]) - const database = yield* Database - const rows = yield* database.execute((db) => db.select().from(auditLogEntries)) - expect(rows.map((row) => row.action)).toEqual(["dashboard.created"]) - }), - ), +/** A warehouse whose `ingest` records each call, or fails every call. */ +const warehouse = (fail = false) => { + const written: Array<{ orgId: string; rows: ReadonlyArray }> = [] + const layer = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: (tenant, _datasource, rows) => + fail + ? Effect.fail( + new WarehouseUpstreamError({ message: "tinybird down", pipeName: "audit_log", cause: new Error("down") }), + ) + : Effect.sync(() => { + // SAFETY: this stub only ever receives the audit datasource's rows. + written.push({ orgId: tenant.orgId, rows: rows as ReadonlyArray }) + }), + }), ) + return { written, layer } +} - // Redelivery is expected — the queue retries whole batches — so a second - // delivery of an already-inserted event must be a no-op, not a duplicate row. - it.effect("is idempotent across redelivery of the same event", () => - run( - Effect.gen(function* () { - const body = event("22222222-2222-4222-8222-222222222222") - yield* processAuditEventsBatch(batchOf(message(body, 1))) - yield* processAuditEventsBatch(batchOf(message(body, 2))) - - const database = yield* Database - const rows = yield* database.execute((db) => db.select().from(auditLogEntries)) - expect(rows).toHaveLength(1) - }), - ), +describe("processAuditEventsBatch", () => { + it.effect("writes well-formed events through ingest, one batch per org, and acks them", () => + Effect.gen(function* () { + const store = warehouse() + const first = message(event("11111111-1111-4111-8111-111111111111"), 1) + const second = message(event("22222222-2222-4222-8222-222222222222"), 1) + const other = message(event("33333333-3333-4333-8333-333333333333", OTHER_ORG), 1) + yield* processAuditEventsBatch(batchOf(first, second, other)).pipe(Effect.provide(store.layer)) + + expect(first.calls).toEqual(["ack"]) + expect(second.calls).toEqual(["ack"]) + expect(other.calls).toEqual(["ack"]) + expect(store.written.map((write) => [write.orgId, write.rows.length]).sort()).toEqual([ + [OTHER_ORG, 1], + [ORG, 2], + ]) + expect(store.written.flatMap((write) => write.rows).every((row) => row.Action === "dashboard.created")).toBe( + true, + ) + }), ) // Cloudflare routes a message to the DLQ only when the consumer retries it // past `max_retries`. Acking on the final attempt would discard the entry // instead, which is exactly the silent drop this branch exists to prevent. - it.effect("retries a failed insert on the final attempt so the message reaches the DLQ", () => + it.effect("retries a failed write on the final attempt so the message reaches the DLQ", () => Effect.gen(function* () { - const exhausted = message(event("33333333-3333-4333-8333-333333333333"), 6) - yield* processAuditEventsBatch(batchOf(exhausted)) - + const exhausted = message(event("44444444-4444-4444-8444-444444444444"), 6) + yield* processAuditEventsBatch(batchOf(exhausted)).pipe(Effect.provide(warehouse(true).layer)) expect(exhausted.calls).toEqual(["retry"]) - }).pipe(Effect.provide(failingDatabase)), + }), ) - it.effect("retries a failed insert while attempts remain", () => + it.effect("retries every message of a failed org batch while attempts remain", () => Effect.gen(function* () { - const failed = message(event("44444444-4444-4444-8444-444444444444"), 2) - yield* processAuditEventsBatch(batchOf(failed)) - - expect(failed.calls).toEqual(["retry"]) - }).pipe(Effect.provide(failingDatabase)), + const a = message(event("55555555-5555-4555-8555-555555555555"), 2) + const b = message(event("66666666-6666-4666-8666-666666666666"), 2) + yield* processAuditEventsBatch(batchOf(a, b)).pipe(Effect.provide(warehouse(true).layer)) + expect(a.calls).toEqual(["retry"]) + expect(b.calls).toEqual(["retry"]) + }), ) // A message that cannot decode will never decode. Retrying only burns the // attempts that would otherwise carry a recoverable message to the DLQ. it.effect("acks a malformed message instead of retrying it forever", () => - run( - Effect.gen(function* () { - const malformed = message({ not: "an audit event" }, 1) - yield* processAuditEventsBatch(batchOf(malformed)) - - expect(malformed.calls).toEqual(["ack"]) - const database = yield* Database - const rows = yield* database.execute((db) => db.select().from(auditLogEntries)) - expect(rows).toEqual([]) - }), - ), + Effect.gen(function* () { + const store = warehouse() + const malformed = message({ not: "an audit event" }, 1) + const fine = message(event("77777777-7777-4777-8777-777777777777"), 1) + yield* processAuditEventsBatch(batchOf(malformed, fine)).pipe(Effect.provide(store.layer)) + + expect(malformed.calls).toEqual(["ack"]) + expect(fine.calls).toEqual(["ack"]) + expect(store.written).toHaveLength(1) + }), ) }) diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts index 8e6c4a926..bb635402e 100644 --- a/apps/api/src/audit-events-runtime.ts +++ b/apps/api/src/audit-events-runtime.ts @@ -1,12 +1,19 @@ -import type { MessageBatch } from "@cloudflare/workers-types" +import type { Message, MessageBatch } from "@cloudflare/workers-types" import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" +import { EdgeCacheService } from "@maple/cache" import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" +import type { OrgId } from "@maple/domain/primitives" import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" -import { auditLogEntries } from "@maple/db" import { Clock, Effect, Layer } from "effect" +import { CacheBackendLive } from "@/platform/CacheBackendLive" import { layerPg } from "@/platform/DatabasePgLive" -import { Database } from "@/platform/DatabaseLive" -import { auditEventToInsert, decodeAuditLogEvent } from "./services/audit/audit-event" +import { Env } from "@/platform/Env" +import { systemTenant } from "@/services/alerts/system-tenant" +import { AUDIT_LOG_DATASOURCE } from "@/services/audit/AuditLogService" +import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" +import { TinybirdOrgTokenService } from "@/services/integrations/TinybirdOrgTokenService" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" +import { type AuditLogEvent, auditEventToRow, decodeAuditLogEvent } from "./services/audit/audit-event" const telemetry = MapleCloudflareSDK.make({ serviceName: "maple-api", @@ -15,9 +22,24 @@ const telemetry = MapleCloudflareSDK.make({ anticipatedErrorIdentifiers: [...ANTICIPATED_ERROR_IDENTIFIERS], }) +/** + * The consumer writes through `WarehouseQueryService.ingest`, which pins every + * write to the managed Tinybird pipeline. The service's read-side dependencies + * (org ClickHouse settings, the per-org JWT minter) come along because the + * layer requires them, not because a write ever consults them. + */ export const buildAuditEventsLayer = (_env: Record) => { + const EnvLive = Env.layer.pipe(Layer.provide(WorkerConfigProviderLayer)) const DatabaseLive = layerPg.pipe(Layer.provide(WorkerEnvironment.layer)) - return DatabaseLive.pipe( + const EdgeCacheServiceLive = EdgeCacheService.layer.pipe(Layer.provide(CacheBackendLive)) + const OrgClickHouseSettingsLive = OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, DatabaseLive, EdgeCacheServiceLive)), + ) + const TinybirdOrgTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(EnvLive)) + const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive, TinybirdOrgTokenLive)), + ) + return WarehouseQueryServiceLive.pipe( Layer.provideMerge(telemetry.layer), Layer.provideMerge(WorkerEnvironment.layer), Layer.provideMerge(WorkerConfigProviderLayer), @@ -27,10 +49,10 @@ export const buildAuditEventsLayer = (_env: Record) => { export const flushAuditEventsTelemetry = (env: Record) => telemetry.flush(env) /** - * Must match `maxRetries` on the audit-events consumer in `alchemy.run.ts` and - * `wrangler.jsonc`. Cloudflare routes the message to the DLQ after this many - * retries without telling us; the check below is what makes the hand-off - * visible in logs at the moment it happens. + * Must match `maxRetries` on the audit-events consumer in `alchemy.run.ts`. + * Cloudflare routes the message to the DLQ after this many retries without + * telling us; the check below is what makes the hand-off visible in logs at + * the moment it happens. */ const AUDIT_EVENTS_MAX_RETRIES = 5 @@ -48,73 +70,99 @@ const auditEventField = (body: unknown, field: string): string => { const auditEventOrgId = (body: unknown) => auditEventField(body, "orgId") const auditEventAction = (body: unknown) => auditEventField(body, "action") +interface DecodedMessage { + readonly message: Message + readonly event: AuditLogEvent +} + +/** + * Retrying past the limit is what hands the message to the DLQ; acking there + * would silently discard it instead. + */ +const retryOrExhaust = (message: Message, cause: unknown) => { + const isFinalAttempt = message.attempts > AUDIT_EVENTS_MAX_RETRIES + return Effect.annotateCurrentSpan({ + "audit.queue.message.outcome": isFinalAttempt ? "exhausted_dlq" : "retry", + }).pipe( + Effect.flatMap(() => + isFinalAttempt + ? Effect.logError("Audit event exhausted retries; routed to dead letter queue").pipe( + Effect.annotateLogs({ + attempt: message.attempts, + orgId: auditEventOrgId(message.body), + action: auditEventAction(message.body), + error: String(cause), + }), + ) + : Effect.logWarning("Audit event write failed; retrying").pipe( + Effect.annotateLogs({ attempt: message.attempts, error: String(cause) }), + ), + ), + Effect.flatMap(() => Effect.sync(() => message.retry())), + ) +} + /** - * Audit events queue consumer: lowers each event to its `audit_log_entries` - * row. The `(org_id, id)` primary key plus `onConflictDoNothing` makes queue - * redelivery idempotent; insert failures retry through the queue's policy and, - * once exhausted, land in `audit-events-dlq` rather than disappearing. + * Audit events queue consumer: lowers each event to its `audit_log` row and + * writes one batch per org through the managed ingest pipeline. The table is a + * ReplacingMergeTree on the entry id, so queue redelivery collapses at merge + * time; write failures retry through the queue's policy and, once exhausted, + * land in `audit-events-dlq` rather than disappearing. */ export const processAuditEventsBatch = (batch: MessageBatch) => Effect.gen(function* () { - const database = yield* Database + const warehouse = yield* WarehouseQueryService + const now = yield* Clock.currentTimeMillis + + const decoded: Array = [] + for (const message of batch.messages) { + const event = yield* decodeAuditLogEvent(message.body).pipe( + Effect.matchEffect({ + // Undecodable now means undecodable on every redelivery, so retrying + // only burns attempts. Acked, but at Error: an audit entry that + // never reaches a row is lost evidence, not routine noise. + onFailure: (error) => + Effect.logError("Discarding malformed audit event queue message").pipe( + Effect.annotateLogs({ attempt: message.attempts, error: String(error) }), + Effect.flatMap(() => Effect.sync(() => message.ack())), + Effect.as(undefined), + ), + onSuccess: (event) => Effect.succeed(event), + }), + ) + if (event !== undefined) decoded.push({ message, event }) + } + + // One `ingest` per org so the write span names the tenant it belongs to. + const byOrg = new Map>() + for (const entry of decoded) { + const group = byOrg.get(entry.event.orgId) + if (group === undefined) byOrg.set(entry.event.orgId, [entry]) + else group.push(entry) + } + yield* Effect.forEach( - batch.messages, - (message) => - decodeAuditLogEvent(message.body).pipe( - Effect.matchEffect({ - // Undecodable now means undecodable on every redelivery, so retrying - // only burns attempts. Acked, but at Error: an audit entry that - // never reaches a row is lost evidence, not routine noise. - onFailure: (error) => - Effect.logError("Discarding malformed audit event queue message").pipe( - Effect.annotateLogs({ attempt: message.attempts, error: String(error) }), - Effect.flatMap(() => Effect.sync(() => message.ack())), - ), - onSuccess: (event) => - Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis - yield* database.execute((db) => - db - .insert(auditLogEntries) - .values(auditEventToInsert(event, now)) - .onConflictDoNothing(), - ) - yield* Effect.sync(() => message.ack()) - }).pipe( - Effect.withSpan("auditEvents.processMessage"), - Effect.catchCause((cause) => { - // Retrying past the limit is what hands the message to the - // DLQ; acking here would silently discard it instead. - const isFinalAttempt = message.attempts > AUDIT_EVENTS_MAX_RETRIES - const outcome = isFinalAttempt ? "exhausted_dlq" : "retry" - return Effect.annotateCurrentSpan({ - "audit.queue.message.outcome": outcome, - }).pipe( - Effect.flatMap(() => - isFinalAttempt - ? Effect.logError( - "Audit event exhausted retries; routed to dead letter queue", - ).pipe( - Effect.annotateLogs({ - attempt: message.attempts, - orgId: auditEventOrgId(message.body), - action: auditEventAction(message.body), - error: String(cause), - }), - ) - : Effect.logWarning("Audit event insert failed; retrying").pipe( - Effect.annotateLogs({ - attempt: message.attempts, - error: String(cause), - }), - ), - ), - Effect.flatMap(() => Effect.sync(() => message.retry())), - ) - }), - ), - }), - ), - { concurrency: 5, discard: true }, + byOrg, + ([orgId, group]) => + warehouse + .ingest( + systemTenant(orgId), + AUDIT_LOG_DATASOURCE, + group.map(({ event }) => auditEventToRow(event, now)), + ) + .pipe( + Effect.flatMap(() => + Effect.sync(() => { + for (const { message } of group) message.ack() + }), + ), + Effect.withSpan("auditEvents.writeOrgBatch", { attributes: { orgId, rows: group.length } }), + Effect.catchCause((cause) => + Effect.forEach(group, ({ message }) => retryOrExhaust(message, cause), { + discard: true, + }), + ), + ), + { concurrency: 3, discard: true }, ) }).pipe(Effect.withSpan("auditEvents.processBatch")) diff --git a/apps/api/src/mcp/dispatcher.test.ts b/apps/api/src/mcp/dispatcher.test.ts index 51ff778fc..0c8fde2fa 100644 --- a/apps/api/src/mcp/dispatcher.test.ts +++ b/apps/api/src/mcp/dispatcher.test.ts @@ -6,6 +6,7 @@ import { MCP_ANTICIPATED_ERROR_IDENTIFIERS } from "./expected-failures" import { mapleToolCatalog, toInputSchema } from "./tools/registry" import type { McpToolRuntimeRequirements } from "./tools/runtime-requirements" import type { TenantContext } from "@/services/auth/tenant-context" +import { AuditLogService, makeMemoryAuditLog } from "@/services/audit/AuditLogService" const TENANT: TenantContext = { orgId: "org_test" as TenantContext["orgId"], @@ -16,7 +17,8 @@ const TENANT: TenantContext = { // These cases stop at registry lookup/schema decoding, before a tool service is read. const makeValidationExecutor = McpToolExecutor.make.pipe( - Effect.provide(Context.empty() as Context.Context), + // Every tool call is audited, so the executor needs the audit service even here. + Effect.provide(Context.make(AuditLogService, makeMemoryAuditLog()) as Context.Context), ) const makeRecordingTracer = () => { diff --git a/apps/api/src/mcp/dispatcher.ts b/apps/api/src/mcp/dispatcher.ts index 687a5340f..e7e306730 100644 --- a/apps/api/src/mcp/dispatcher.ts +++ b/apps/api/src/mcp/dispatcher.ts @@ -7,6 +7,7 @@ import type { McpToolRuntimeRequirements } from "./tools/runtime-requirements" import { CurrentMcpTenant } from "./lib/query-warehouse" import { recordExpectedMcpFailure } from "./expected-failures" import type { TenantContext } from "@/services/auth/tenant-context" +import { recordMcpToolAudit } from "@/services/audit/audit-access" /** * Built on first use, not at module scope. @@ -172,10 +173,20 @@ export class McpToolExecutor extends Context.Service( warehouse, ) + const audit = (result: Parameters[0]["result"]) => + recordRawSqlAudit({ + tenant: input.tenant, + sql: input.sql, + context: "mcp.run_sql", + startTime: input.startTime, + endTime: input.endTime, + result, + }) return yield* executeRawSql(input.tenant, { sql: input.sql, orgId: input.tenant.orgId, @@ -50,5 +60,15 @@ export const runRawSql = Effect.fn("runRawSql")(function* (input: RunRawSqlInput granularitySeconds: input.granularitySeconds, workload: "interactive", context: "mcp.run_sql", - }) + }).pipe( + // Every statement is audited, however it ended: a refused one as `denied`. + Effect.tap((result) => audit({ _tag: "rows", rowCount: result.rowCount })), + Effect.tapError((error) => + audit( + error._tag === "@maple/http/errors/RawSqlValidationError" + ? { _tag: "rejected", reason: error.message } + : { _tag: "failed", error: describeFailure(error) }, + ), + ), + ) }) diff --git a/apps/api/src/platform/time.ts b/apps/api/src/platform/time.ts index e4dae4216..6b22ad5c8 100644 --- a/apps/api/src/platform/time.ts +++ b/apps/api/src/platform/time.ts @@ -32,3 +32,14 @@ export function dateToMs(date: Date | null | undefined): number | null export function dateToMs(date: Date | null | undefined): number | null { return date === null || date === undefined ? null : date.getTime() } + +/** + * Tinybird `DateTime64(3)` wire format for `ingest` rows: `YYYY-MM-DD HH:mm:ss.SSS`, + * UTC, no zone. Every direct warehouse write (alert checks, audit entries) sends + * timestamps this way; ISO's `T`/`Z` is rejected by the Events API JSONPath parser. + */ +export function msToWarehouseDateTime64(ms: number): string { + const d = new Date(ms) + const pad = (n: number, w = 2) => n.toString().padStart(w, "0") + return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` +} diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 2c2cd8213..d3498c8bb 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -90,6 +90,7 @@ import { Clock, Effect, Match, Option, Schema } from "effect" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import { isMissingProductEvents, isMissingServiceOperationsRollup } from "@/services/warehouse/missing-table" import { makeDirectRouteCachePolicy, makeExecuteRawSql } from "@maple/query-engine/runtime" +import { describeFailure, recordRawSqlAudit } from "@/services/audit/audit-access" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { traceCacheTtlSeconds } from "@/services/warehouse/trace-detail-cache" import { @@ -2073,6 +2074,15 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query const autoBucketSeconds = computeAutoBucketSeconds(payload.startTime, payload.endTime) const granularitySeconds = payload.granularitySeconds ?? autoBucketSeconds + const audit = (result: Parameters[0]["result"]) => + recordRawSqlAudit({ + tenant, + sql: payload.sql, + context: "rawSql", + startTime: payload.startTime, + endTime: payload.endTime, + result, + }) const result = yield* mapExecError( executeRawSql(tenant, { sql: payload.sql, @@ -2082,7 +2092,17 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query granularitySeconds, workload: "interactive", context: "rawSql", - }), + }).pipe( + // Every statement is audited, however it ended: a refused one as `denied`. + Effect.tap((executed) => audit({ _tag: "rows", rowCount: executed.rowCount })), + Effect.tapError((error) => + audit( + error._tag === "@maple/http/errors/RawSqlValidationError" + ? { _tag: "rejected", reason: error.message } + : { _tag: "failed", error: describeFailure(error) }, + ), + ), + ), "rawSql query failed", ) diff --git a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts index 28f4a286e..201d2daee 100644 --- a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts +++ b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts @@ -175,7 +175,7 @@ const makeHarness = () => { Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index da3466686..4e84f09ad 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -165,7 +165,7 @@ const makeHarness = ( Layer.provide(SlackIntegrationServiceStubLayer), Layer.provide(PlanetScaleServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/api-keys.http.test.ts b/apps/api/src/routes/v2/api-keys.http.test.ts index 6e6ae3a27..2b2500e84 100644 --- a/apps/api/src/routes/v2/api-keys.http.test.ts +++ b/apps/api/src/routes/v2/api-keys.http.test.ts @@ -68,7 +68,7 @@ const makeHarness = (checkRateLimit: RateLimiterApi["check"] = () => Effect.succ Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(Layer.succeed(ApiV2RateLimiter, { check: checkRateLimit })), Layer.provideMerge(servicesLive), Layer.provideMerge(HttpRouter.cors(API_CORS_OPTIONS)), diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts index ec9b351a7..450b89e04 100644 --- a/apps/api/src/routes/v2/audit-log.http.ts +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -1,5 +1,5 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" -import { AuditChanges, CurrentTenant } from "@maple/domain/http" +import { CurrentTenant } from "@maple/domain/http" import { ActorId, ApiKeyId, UserId } from "@maple/domain/primitives" import { decodePublicId, @@ -12,7 +12,7 @@ import { V2ParameterInvalid, } from "@maple/domain/http/v2" import type { V2AuditLogEntry } from "@maple/domain/http/v2" -import type { AuditLogEntryRow } from "@maple/db" +import type { AuditLogEntry } from "@/services/audit/audit-event" import { Effect, Option, Schema } from "effect" import { AuditLogService } from "@/services/audit/AuditLogService" import { requireAdmin } from "@/services/auth/auth" @@ -54,13 +54,8 @@ const actorIdentityFilter = (publicActorId: string) => { }) } -const isJsonRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeChangesOption = Schema.decodeUnknownOption(AuditChanges) - /** The actor's public identifier, matching the ID style of its own resource. */ -const publicActorId = (row: AuditLogEntryRow): string | null => { +const publicActorId = (row: AuditLogEntry): string | null => { switch (row.actorType) { case "api_key": return row.apiKeyId === null ? null : encodePublicId(PublicIdPrefixes.apiKey, row.apiKeyId) @@ -74,7 +69,7 @@ const publicActorId = (row: AuditLogEntryRow): string | null => { } } -const toV2AuditLogEntry = (row: AuditLogEntryRow): V2AuditLogEntry => ({ +const toV2AuditLogEntry = (row: AuditLogEntry): V2AuditLogEntry => ({ id: row.id, object: "audit_log_entry", action: row.action, @@ -87,8 +82,8 @@ const toV2AuditLogEntry = (row: AuditLogEntryRow): V2AuditLogEntry => ({ source: row.source, resource_type: row.resourceType, resource_id: row.resourceId, - changes: Option.getOrNull(decodeChangesOption(row.changesJson)), - metadata: isJsonRecord(row.metadataJson) ? row.metadataJson : null, + changes: row.changes, + metadata: row.metadata, request_id: row.requestId, origin_ip: row.originIp, origin_country: row.originCountry, diff --git a/apps/api/src/routes/v2/config-resources.http.test.ts b/apps/api/src/routes/v2/config-resources.http.test.ts index 7d20c7288..1daca7a6c 100644 --- a/apps/api/src/routes/v2/config-resources.http.test.ts +++ b/apps/api/src/routes/v2/config-resources.http.test.ts @@ -116,7 +116,7 @@ const makeHarness = () => { // session_replays (in AllV2GroupLayersLive) needs the warehouse at the routes level. Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/dashboards.http.test.ts b/apps/api/src/routes/v2/dashboards.http.test.ts index 1418b1fa3..2a414c511 100644 --- a/apps/api/src/routes/v2/dashboards.http.test.ts +++ b/apps/api/src/routes/v2/dashboards.http.test.ts @@ -66,7 +66,7 @@ const makeHarness = () => { Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/integrations.http.test.ts b/apps/api/src/routes/v2/integrations.http.test.ts index ea54c4b3a..e9d04f3a3 100644 --- a/apps/api/src/routes/v2/integrations.http.test.ts +++ b/apps/api/src/routes/v2/integrations.http.test.ts @@ -171,7 +171,7 @@ const makeHarness = (slack: Partial = {}, planetscal Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/mobile-devices.http.test.ts b/apps/api/src/routes/v2/mobile-devices.http.test.ts index c88e67c07..86b2c14a1 100644 --- a/apps/api/src/routes/v2/mobile-devices.http.test.ts +++ b/apps/api/src/routes/v2/mobile-devices.http.test.ts @@ -80,7 +80,7 @@ const makeHarness = () => { Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/phase1-resources.http.test.ts b/apps/api/src/routes/v2/phase1-resources.http.test.ts index fa9a51b64..1531bb353 100644 --- a/apps/api/src/routes/v2/phase1-resources.http.test.ts +++ b/apps/api/src/routes/v2/phase1-resources.http.test.ts @@ -573,7 +573,7 @@ const makeHarness = ( Layer.provide(ConfigResourceServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/setup-audit.http.test.ts b/apps/api/src/routes/v2/setup-audit.http.test.ts index ad678e8a4..e3c6a7247 100644 --- a/apps/api/src/routes/v2/setup-audit.http.test.ts +++ b/apps/api/src/routes/v2/setup-audit.http.test.ts @@ -143,7 +143,7 @@ const makeHarness = (warehouse: WarehouseQueryServiceApi = warehouseStub()) => { Layer.provide(TelemetryServiceStubsLayer), Layer.provide(warehouseLive), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts index 68bd4bc11..c59e59eaf 100644 --- a/apps/api/src/routes/v2/telemetry.http.test.ts +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -276,7 +276,7 @@ const makeHarness = ( Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 054eba799..3d397f6dd 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -79,7 +79,7 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, // Real service, no stub: it needs only the Database every harness already provides. - HttpV2AuditLogLive.pipe(Layer.provide(AuditLogService.layer)), + HttpV2AuditLogLive.pipe(Layer.provide(AuditLogService.layerMemory)), HttpV2ScrapeTargetsLive, HttpV2InstrumentationRecommendationsLive, HttpV2InstrumentationAuditLive, @@ -125,7 +125,7 @@ export const AllV2GroupLayersLive = Layer.mergeAll( ).pipe( // Mutation handlers across the groups record audit entries; the real service // needs only the Database every harness already provides. - Layer.provide(AuditLogService.layer), + Layer.provide(AuditLogService.layerMemory), ) export const ApiV2RateLimiterAllowAllLayer = Layer.succeed(ApiV2RateLimiter, { diff --git a/apps/api/src/routes/v2/widget-credentials.http.test.ts b/apps/api/src/routes/v2/widget-credentials.http.test.ts index c09ec82be..b785222b2 100644 --- a/apps/api/src/routes/v2/widget-credentials.http.test.ts +++ b/apps/api/src/routes/v2/widget-credentials.http.test.ts @@ -78,7 +78,7 @@ const makeHarness = () => { Layer.provide(PlanetScaleServiceStubsLayer), Layer.provide(TelemetryServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/v2/widget-summary.http.test.ts b/apps/api/src/routes/v2/widget-summary.http.test.ts index d7ca174f3..fc25fb99b 100644 --- a/apps/api/src/routes/v2/widget-summary.http.test.ts +++ b/apps/api/src/routes/v2/widget-summary.http.test.ts @@ -191,7 +191,7 @@ const makeHarness = (options: { Layer.provide(AlertsServiceStubLayer), Layer.provide(ConfigResourceServiceStubsLayer), Layer.provideMerge(ApiAuthorizationV2Layer), - Layer.provideMerge(AuditLogService.layer), + Layer.provideMerge(AuditLogService.layerMemory), Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), Layer.provideMerge(servicesLive), ) diff --git a/apps/api/src/routes/webhooks/webhooks.http.test.ts b/apps/api/src/routes/webhooks/webhooks.http.test.ts index 3d097177a..1af291049 100644 --- a/apps/api/src/routes/webhooks/webhooks.http.test.ts +++ b/apps/api/src/routes/webhooks/webhooks.http.test.ts @@ -197,9 +197,10 @@ describe("ClerkWebhookRouter", () => { const audit = recordingAudit() const configured = HttpRouter.toWebHandler( makeRouterLayer( - ClerkWebhookRouter, + ClerkWebhookRoute, { CLERK_WEBHOOK_SECRET: CLERK_SECRET }, events.layer, + recordingRevocation().layer, audit.layer, ), { disableLogger: true }, diff --git a/apps/api/src/runtime/graph-boundaries.test.ts b/apps/api/src/runtime/graph-boundaries.test.ts index 70f3a3051..03ee0b1b2 100644 --- a/apps/api/src/runtime/graph-boundaries.test.ts +++ b/apps/api/src/runtime/graph-boundaries.test.ts @@ -72,7 +72,7 @@ describe("API runtime graph boundaries", () => { "AlertRulesServiceLive", "AlertsServiceLive", // Lets `register_agent` (and issue-workflow mutations) write org audit entries. - "AuditLogService.layer", + "AuditLogServiceLive", "DashboardPersistenceService.layer", "ErrorActorsServiceLive", "ErrorIssueReadModelsServiceLive", diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 64e91fe60..211ab94c2 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -50,7 +50,7 @@ import { HttpV2MobileDevicesLive } from "@/routes/v2/mobile-devices.http" import { HttpV2OrganizationLive } from "@/routes/v2/organization.http" import { HttpV2InstrumentationRecommendationsLive } from "@/routes/v2/recommendations.http" import { HttpV2AuditLogLive } from "@/routes/v2/audit-log.http" -import { AuditLogService } from "@/services/audit/AuditLogService" +import { AuditLogServiceLive } from "@/runtime/service-graph" import { HttpV2ScrapeTargetsLive } from "@/routes/v2/scrape-targets.http" import { HttpV2InstrumentationAuditLive } from "@/routes/v2/setup-audit.http" import { HttpV2SessionReplaysLive } from "@/routes/v2/session-replays.http" @@ -189,8 +189,8 @@ export const ApiAuthLive = Layer.mergeAll( Layer.provideMerge(ApiV2RateLimiter.layer), Layer.provideMerge(McpToolRateLimiter.layer), Layer.provideMerge(ApiKeysService.layer), - // Denied attempts are audited from inside the auth layers themselves. - Layer.provideMerge(AuditLogService.layer), + // Denied attempts and audited reads are recorded from inside the auth layers. + Layer.provideMerge(AuditLogServiceLive), // Membership verification for `x-maple-org-id`. Only the v2 layer asks for // it; without it that layer cannot build, which is deliberate — the header // must never end up silently ignored in a runtime that forgot to wire this. diff --git a/apps/api/src/runtime/mcp-service-graph.ts b/apps/api/src/runtime/mcp-service-graph.ts index 4adf4dda6..b5af3a326 100644 --- a/apps/api/src/runtime/mcp-service-graph.ts +++ b/apps/api/src/runtime/mcp-service-graph.ts @@ -51,6 +51,8 @@ const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( Layer.provide(Layer.mergeAll(InfraLive, OrgClickHouseSettingsServiceLive, TinybirdOrgTokenServiceLive)), ) +const AuditLogServiceLive = AuditLogService.layer.pipe(Layer.provide(WarehouseQueryServiceLive)) + const BucketCacheServiceLive = BucketCacheService.layer.pipe(Layer.provideMerge(EdgeCacheServiceLive)) const QueryEngineServiceLive = QueryEngineService.layer.pipe( @@ -105,7 +107,7 @@ const NotificationDispatcherLive = NotificationDispatcher.layer.pipe( const ErrorActorsServiceLive = ErrorActorsService.layer const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(Layer.mergeAll(ErrorActorsServiceLive, AuditLogService.layer)), + Layer.provide(Layer.mergeAll(ErrorActorsServiceLive, AuditLogServiceLive)), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer const ErrorIssueReadModelsServiceLive = ErrorIssueReadModelsService.layer.pipe( @@ -166,7 +168,7 @@ const McpRuntimeServicesLive = Layer.mergeAll( AlertReadModelsServiceLive, AlertRulesServiceLive, AlertsServiceLive, - AuditLogService.layer, + AuditLogServiceLive, DashboardPersistenceService.layer, ErrorActorsServiceLive, ErrorIssueReadModelsServiceLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 8f263e7d4..7ccf292a6 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -85,7 +85,6 @@ const EdgeCacheServiceLive = EdgeCacheService.layer.pipe(Layer.provide(CacheBack const CoreServicesLive = Layer.mergeAll( AuthService.layer, ApiKeysService.layer, - AuditLogService.layer, CliDeviceAuthService.layer, McpOAuthService.layer, CloudflareOAuthService.layer, @@ -112,6 +111,13 @@ const CoreServicesLive = Layer.mergeAll( const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe(Layer.provideMerge(CoreServicesLive)) +/** + * Audit entries are warehouse rows (Tinybird-pinned `ingest`), so the service + * composes after the warehouse rather than inside CoreServicesLive. Exported + * for the auth layers in `http-graph.ts`, which record denials and reads. + */ +export const AuditLogServiceLive = AuditLogService.layer.pipe(Layer.provide(WarehouseQueryServiceLive)) + // Serves the integration page's per-zone collection status; the poll loop itself // runs in the alerting worker's cron, not here. const CloudflareAnalyticsServiceLive = CloudflareAnalyticsService.layer.pipe( @@ -178,7 +184,7 @@ const NotificationDispatcherLive = NotificationDispatcher.layer.pipe( const ErrorActorsServiceLive = ErrorActorsService.layer const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(AuditLogService.layer), + Layer.provide(AuditLogServiceLive), Layer.provideMerge(ErrorActorsServiceLive), ) const ErrorPolicyServiceLive = ErrorPolicyService.layer @@ -302,6 +308,7 @@ const MainServicesLive = Layer.mergeAll( ProductEventsServiceLive, DailySpendServiceLive, CloudflareAnalyticsServiceLive, + AuditLogServiceLive, WarehouseQueryServiceLive, EdgeCacheServiceLive, QueryEngineServiceLive, diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 49fccdfb4..019c36116 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -89,7 +89,7 @@ import { EmailService } from "@/platform/EmailService" import { Env } from "@/platform/Env" import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" import { makeDbExecute } from "@/platform/db-execute" -import { dateToMs, msToDate, msToSqlTimestamp } from "@/platform/time" +import { dateToMs, msToDate, msToSqlTimestamp, msToWarehouseDateTime64 } from "@/platform/time" import { makePersistenceError } from "./alert-persistence" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import type { GroupedAlertObservation } from "@maple/query-engine/runtime" @@ -283,13 +283,7 @@ export const interleaveAlertRulesByOrg = ( return fair } -// Tinybird DateTime64(3) wire format for alert_checks ingest: -// "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). -const toIngestDateTime64 = (epochMs: number) => { - const d = new Date(epochMs) - const pad = (n: number, w = 2) => n.toString().padStart(w, "0") - return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` -} +const toIngestDateTime64 = msToWarehouseDateTime64 const compareThreshold = ( value: number, diff --git a/apps/api/src/services/audit/AuditLogService.test.ts b/apps/api/src/services/audit/AuditLogService.test.ts index ddd3a8ca2..0dc5b7b10 100644 --- a/apps/api/src/services/audit/AuditLogService.test.ts +++ b/apps/api/src/services/audit/AuditLogService.test.ts @@ -1,28 +1,242 @@ -import { afterEach, describe, expect, it } from "@effect/vitest" +import { describe, expect, it } from "@effect/vitest" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { CurrentTenant } from "@maple/domain/http" import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" -import { OrgId, UserId } from "@maple/domain/primitives" +import { ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" +import type { AuditLogRow } from "@maple/domain/tinybird" import { Effect, Layer, Schema } from "effect" import { TestClock } from "effect/testing" -import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" -import { AuditLogService, recordHttpAudit } from "./AuditLogService" -import { CurrentTenant } from "@maple/domain/http" -import { ApiKeyId } from "@maple/domain/primitives" +import { makeWarehouseServiceStub } from "@/routes/v2/v2-test-support" import { type AuditActorInfo, CurrentAuditActor } from "@/services/auth/audit-actor" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" +import { AUDIT_LOG_DATASOURCE, AuditLogService, recordHttpAudit } from "./AuditLogService" const asOrgId = Schema.decodeUnknownSync(OrgId) const asUserId = Schema.decodeUnknownSync(UserId) const ORG = asOrgId("org_audit_log_test") const USER = asUserId("user_audit_log_test") -const createdDbs: TestDb[] = [] - -afterEach(() => cleanupTestDbs(createdDbs)) - const DASHBOARD_ID = "3f1b7c02-9a44-4d1e-8b2f-0c5d6e7a8b91" const API_KEY = Schema.decodeUnknownSync(ApiKeyId)("7b2e4c10-55aa-4d3e-9f21-1a2b3c4d5e6f") -const makeLayer = () => AuditLogService.layer.pipe(Layer.provide(createTestDb(createdDbs).layer)) +/** + * A warehouse that records what `ingest` receives and answers `compiledQuery` + * with canned rows, exposing the SQL it was handed so a test can assert which + * filters the listing bound. + */ +const recordingWarehouse = (rows: ReadonlyArray> = []) => { + const ingested: Array<{ datasource: string; rows: ReadonlyArray }> = [] + const sql: Array = [] + const layer = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: (_tenant, datasource, batch) => + Effect.sync(() => { + // SAFETY: this stub only ever receives the audit datasource's rows. + ingested.push({ datasource, rows: batch as ReadonlyArray }) + }), + compiledQuery: ((_tenant: unknown, compiled: unknown) => + Effect.gen(function* () { + const query = Effect.isEffect(compiled) ? yield* compiled : compiled + // SAFETY: every compiled query carries its SQL text. + sql.push((query as { readonly sql: string }).sql) + return rows + })) as never, + }), + ) + return { ingested, sql, layer } +} + +const storedRow = (overrides: Partial> = {}) => ({ + id: "9d2c1e3a-6a1b-4f0e-9c1d-2b3a4c5d6e7f", + occurredAt: "2026-08-29 09:12:00.412", + recordedAt: "2026-08-29 09:12:00.900", + actorType: "user", + userId: USER, + apiKeyId: "", + actorId: "", + actorLabel: "David", + affectedUserId: "", + source: "dashboard", + action: "dashboard.updated", + outcome: "allowed", + denialReason: "", + resourceType: "dashboard", + resourceId: "dash_1", + changedFields: ["name"], + changes: JSON.stringify({ fields: ["name"], before: { name: "a" }, after: { name: "b" } }), + metadata: JSON.stringify({ reason: "rename" }), + requestId: "ray", + originIp: "203.0.113.7", + originCountry: "DE", + ...overrides, +}) + +describe("AuditLogService (warehouse-backed)", () => { + it.effect("writes one audit_log row through ingest, with '' for absent values", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + // Internal ID in, public `dash_…` ID out — the service owns the encoding. + resourceId: DASHBOARD_ID, + metadata: { name: "First" }, + }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + + expect(warehouse.ingested).toHaveLength(1) + expect(warehouse.ingested[0]!.datasource).toBe(AUDIT_LOG_DATASOURCE) + const row = warehouse.ingested[0]!.rows[0]! + expect(row.OrgId).toBe(ORG) + expect(row.ActorType).toBe("user") + expect(row.UserId).toBe(USER) + expect(row.ApiKeyId).toBe("") + expect(row.ResourceType).toBe("dashboard") + expect(row.ResourceId).toBe(encodePublicId(PublicIdPrefixes.dashboard, DASHBOARD_ID)) + expect(row.ChangedFields).toEqual([]) + expect(row.Changes).toBe("") + expect(JSON.parse(row.Metadata)).toEqual({ name: "First" }) + expect(row.OccurredAt).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/) + }), + ) + + it.effect("publishes to the audit queue instead of writing when the binding is present", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + const sent: unknown[] = [] + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + }) + }).pipe( + Effect.provide( + AuditLogService.layer.pipe( + Layer.provide(warehouse.layer), + Layer.provide( + Layer.succeed(WorkerEnvironment, { + AUDIT_EVENTS_QUEUE: { + send: async (message: unknown) => { + sent.push(message) + }, + }, + }), + ), + ), + ), + ) + expect(sent).toHaveLength(1) + expect(sent[0]).toMatchObject({ orgId: ORG, action: "dashboard.created" }) + // The consumer performs the write; nothing reaches the warehouse directly. + expect(warehouse.ingested).toEqual([]) + }), + ) + + it.effect("degrades to a direct write when the queue send fails", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "api_key" }, + source: "api", + action: "alert_rule.updated", + }) + }).pipe( + Effect.provide( + AuditLogService.layer.pipe( + Layer.provide(warehouse.layer), + Layer.provide( + Layer.succeed(WorkerEnvironment, { + AUDIT_EVENTS_QUEUE: { + send: async () => { + throw new Error("broker down") + }, + }, + }), + ), + ), + ), + ) + expect(warehouse.ingested).toHaveLength(1) + expect(warehouse.ingested[0]!.rows[0]!.Action).toBe("alert_rule.updated") + }), + ) + + it.effect("decodes stored rows: '' becomes null, documents parse, timestamps are UTC", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse([storedRow()]) + const rows = yield* Effect.gen(function* () { + const audit = yield* AuditLogService + return yield* audit.list(ORG, { limit: 10, offset: 0 }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + + expect(rows).toHaveLength(1) + const entry = rows[0]! + expect(entry.orgId).toBe(ORG) + expect(entry.userId).toBe(USER) + expect(entry.apiKeyId).toBeNull() + expect(entry.affectedUserId).toBeNull() + expect(entry.denialReason).toBeNull() + expect(entry.changedFields).toEqual(["name"]) + expect(entry.changes).toEqual({ fields: ["name"], before: { name: "a" }, after: { name: "b" } }) + expect(entry.metadata).toEqual({ reason: "rename" }) + expect(entry.occurredAt.toISOString()).toBe("2026-08-29T09:12:00.412Z") + expect(entry.recordedAt.toISOString()).toBe("2026-08-29T09:12:00.900Z") + }), + ) + + it.effect("an entry without a diff lists with null changed fields", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse([storedRow({ changes: "", changedFields: [], metadata: "" })]) + const rows = yield* Effect.gen(function* () { + const audit = yield* AuditLogService + return yield* audit.list(ORG, { limit: 10, offset: 0 }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + expect(rows[0]!.changes).toBeNull() + expect(rows[0]!.changedFields).toBeNull() + expect(rows[0]!.metadata).toBeNull() + }), + ) + + it.effect("binds only the filters the caller set", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.list(ORG, { limit: 10, offset: 0 }) + yield* audit.list(ORG, { + actorType: "api_key", + changedField: "scopes", + sinceMs: Date.UTC(2026, 7, 29, 9, 12, 0, 412), + limit: 5, + offset: 5, + }) + }).pipe(Effect.provide(AuditLogService.layer.pipe(Layer.provide(warehouse.layer)))) + + const [plain, filtered] = warehouse.sql + expect(plain).toContain(`OrgId = '${ORG}'`) + expect(plain).not.toContain("ActorType =") + expect(plain).not.toContain("has(ChangedFields") + expect(plain).toMatch(/ORDER BY occurredAt DESC, id DESC/) + expect(filtered).toContain("ActorType = 'api_key'") + expect(filtered).toContain("has(ChangedFields, 'scopes')") + expect(filtered).toContain("OccurredAt >= '2026-08-29 09:12:00.412'") + expect(filtered).toMatch(/LIMIT 5\s+OFFSET 5/) + // Never routed to an org's BYO warehouse. + expect(plain).toContain("audit_log") + }), + ) +}) /** Three entries with distinct timestamps: user, then api_key, then agent. */ const seedThree = Effect.gen(function* () { @@ -32,7 +246,6 @@ const seedThree = Effect.gen(function* () { actor: { type: "user", userId: USER }, source: "dashboard", action: "dashboard.created", - // Internal ID in, public `dash_…` ID out — the service owns the encoding. resourceId: DASHBOARD_ID, metadata: { name: "First" }, }) @@ -52,7 +265,9 @@ const seedThree = Effect.gen(function* () { }) }) -describe("AuditLogService", () => { +// The in-memory layer backs every route and workflow test, so its filter and +// ordering semantics must match the warehouse query's. +describe("AuditLogService.layerMemory", () => { it.effect("round-trips a recorded entry and lists newest first", () => Effect.gen(function* () { const audit = yield* AuditLogService @@ -71,44 +286,15 @@ describe("AuditLogService", () => { expect(oldest.source).toBe("dashboard") expect(oldest.resourceType).toBe("dashboard") expect(oldest.resourceId).toBe(encodePublicId(PublicIdPrefixes.dashboard, DASHBOARD_ID)) - expect(oldest.metadataJson).toEqual({ name: "First" }) + expect(oldest.metadata).toEqual({ name: "First" }) const newest = rows[0]! expect(newest.actorType).toBe("agent") expect(newest.actorLabel).toBe("triage-bot") - }).pipe(Effect.provide(makeLayer())), - ) - - it.effect("filters by actor type", () => - Effect.gen(function* () { - const audit = yield* AuditLogService - yield* seedThree - - const apiKeyRows = yield* audit.list(ORG, { actorType: "api_key", limit: 10, offset: 0 }) - expect(apiKeyRows.map((row) => row.action)).toEqual(["alert_rule.updated"]) - - const systemRows = yield* audit.list(ORG, { actorType: "system", limit: 10, offset: 0 }) - expect(systemRows).toEqual([]) - }).pipe(Effect.provide(makeLayer())), - ) - - it.effect("pages with offset and limit in newest-first order", () => - Effect.gen(function* () { - const audit = yield* AuditLogService - yield* seedThree - - const firstPage = yield* audit.list(ORG, { limit: 2, offset: 0 }) - expect(firstPage.map((row) => row.action)).toEqual([ - "error_issue.state_change", - "alert_rule.updated", - ]) - - const secondPage = yield* audit.list(ORG, { limit: 2, offset: 2 }) - expect(secondPage.map((row) => row.action)).toEqual(["dashboard.created"]) - }).pipe(Effect.provide(makeLayer())), + }).pipe(Effect.provide(AuditLogService.layerMemory)), ) - it.effect("records denied outcomes and filters by outcome", () => + it.effect("filters by actor type and outcome, and pages newest-first", () => Effect.gen(function* () { const audit = yield* AuditLogService yield* seedThree @@ -121,14 +307,17 @@ describe("AuditLogService", () => { denialReason: "missing role: admin", }) + const apiKeyRows = yield* audit.list(ORG, { actorType: "api_key", limit: 10, offset: 0 }) + expect(apiKeyRows.map((row) => row.action)).toEqual(["alert_rule.updated"]) + expect(yield* audit.list(ORG, { actorType: "system", limit: 10, offset: 0 })).toEqual([]) + const denied = yield* audit.list(ORG, { outcome: "denied", limit: 10, offset: 0 }) expect(denied.map((row) => row.action)).toEqual(["alert_rule.deleted"]) - expect(denied[0]!.outcome).toBe("denied") expect(denied[0]!.denialReason).toBe("missing role: admin") - const allowed = yield* audit.list(ORG, { outcome: "allowed", limit: 10, offset: 0 }) - expect(allowed).toHaveLength(3) - }).pipe(Effect.provide(makeLayer())), + const secondPage = yield* audit.list(ORG, { limit: 2, offset: 2 }) + expect(secondPage.map((row) => row.action)).toEqual(["alert_rule.updated", "dashboard.created"]) + }).pipe(Effect.provide(AuditLogService.layerMemory)), ) it.effect("stores update diffs and filters by changed field", () => @@ -146,50 +335,11 @@ describe("AuditLogService", () => { const rows = yield* audit.list(ORG, { changedField: "name", limit: 10, offset: 0 }) expect(rows.map((row) => row.action)).toEqual(["dashboard.updated"]) expect(rows[0]!.changedFields).toEqual(["name"]) - expect(rows[0]!.changesJson).toEqual({ - fields: ["name"], - before: { name: "a" }, - after: { name: "b" }, - }) - - const none = yield* audit.list(ORG, { changedField: "description", limit: 10, offset: 0 }) - expect(none).toEqual([]) - }).pipe(Effect.provide(makeLayer())), + expect(rows[0]!.changes).toEqual({ fields: ["name"], before: { name: "a" }, after: { name: "b" } }) + expect(yield* audit.list(ORG, { changedField: "description", limit: 10, offset: 0 })).toEqual([]) + }).pipe(Effect.provide(AuditLogService.layerMemory)), ) - it.effect("publishes to the audit queue instead of writing when the binding is present", () => { - const sent: unknown[] = [] - return Effect.gen(function* () { - const audit = yield* AuditLogService - yield* audit.record({ - orgId: ORG, - actor: { type: "user", userId: USER }, - source: "dashboard", - action: "dashboard.created", - }) - - expect(sent).toHaveLength(1) - expect(sent[0]).toMatchObject({ orgId: ORG, action: "dashboard.created" }) - // The consumer performs the insert; nothing lands in the DB directly. - const rows = yield* audit.list(ORG, { limit: 10, offset: 0 }) - expect(rows).toEqual([]) - }).pipe( - Effect.provide( - makeLayer().pipe( - Layer.provide( - Layer.succeed(WorkerEnvironment, { - AUDIT_EVENTS_QUEUE: { - send: async (message: unknown) => { - sent.push(message) - }, - }, - }), - ), - ), - ), - ) - }) - // The credential and the surface are the two facts a mutation handler cannot // re-derive, and getting them wrong is what made API-key and MCP actions read // back as dashboard sessions. @@ -210,7 +360,7 @@ describe("AuditLogService", () => { }).pipe( Effect.provideService(CurrentTenant.Context, tenant), Effect.provideService(CurrentAuditActor, info), - Effect.provide(makeLayer().pipe(Layer.provide(Layer.succeed(WorkerEnvironment, {})))), + Effect.provide(AuditLogService.layerMemory), ) it.effect("attributes an API-key request to the key, not the dashboard", () => @@ -238,8 +388,6 @@ describe("AuditLogService", () => { }), ) - // Requests that skipped every auth middleware still have a tenant; the - // fallback must not invent a credential it did not see. it.effect("falls back to the tenant user when no middleware set the reference", () => Effect.gen(function* () { const row = yield* recordAs(undefined) @@ -250,21 +398,4 @@ describe("AuditLogService", () => { }), ) }) - - it.effect("writes directly when the queue binding is absent from the worker environment", () => - Effect.gen(function* () { - const audit = yield* AuditLogService - yield* audit.record({ - orgId: ORG, - actor: { type: "user", userId: USER }, - source: "dashboard", - action: "dashboard.created", - }) - - const rows = yield* audit.list(ORG, { limit: 10, offset: 0 }) - expect(rows.map((row) => row.action)).toEqual(["dashboard.created"]) - }).pipe( - Effect.provide(makeLayer().pipe(Layer.provide(Layer.succeed(WorkerEnvironment, {})))), - ), - ) }) diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index fbce6c231..dd53852fe 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -4,16 +4,24 @@ import { AuditLogPersistenceError, CurrentTenant } from "@maple/domain/http" import type { AuditActorType, AuditChanges, AuditLogSource, AuditOutcome } from "@maple/domain/http" import type { ActorId, ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" import { AuditLogEntryId as AuditLogEntryIdSchema } from "@maple/domain/primitives" -import { auditLogEntries, type AuditLogEntryRow } from "@maple/db" -import { and, arrayContains, desc, eq, gte, lte } from "drizzle-orm" +import * as CH from "@maple/query-engine/ch" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import type { Queue } from "@cloudflare/workers-types" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" -import { Database, type DatabaseError } from "@/platform/DatabaseLive" -import { msToDate } from "@/platform/time" +import { msToWarehouseDateTime64 } from "@/platform/time" +import { systemTenant } from "@/services/alerts/system-tenant" import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { type AuditAction, auditResourceFields, type AuditResourceIdOption } from "./audit-actions" -import { AuditLogEvent, auditEventToInsert, encodeAuditLogEventSync } from "./audit-event" +import { + AuditLogEvent, + type AuditLogEntry, + auditEventToEntry, + auditEventToRow, + decodeStoredAuditLogEntry, + encodeAuditLogEventSync, + storedRowToEntry, +} from "./audit-event" const decodeAuditLogEntryIdSync = Schema.decodeUnknownSync(AuditLogEntryIdSchema) @@ -28,15 +36,18 @@ class AuditQueueSendError extends Schema.TaggedError()( /** Producer binding name; the paired `*_NAME` var drives consumer dispatch. */ export const AUDIT_EVENTS_QUEUE_BINDING = "AUDIT_EVENTS_QUEUE" +/** The warehouse datasource every audit entry lands in. */ +export const AUDIT_LOG_DATASOURCE = "audit_log" + /** - * `queue.send` sits on the response path of every mutation and denial. A - * healthy send is tens of ms; 2s bounds a stalling broker before the entry - * degrades to a direct Postgres write. + * `queue.send` sits on the response path of every mutation, denial, and + * audited read. A healthy send is tens of ms; 2s bounds a stalling broker + * before the entry degrades to a direct warehouse write. */ export const AUDIT_QUEUE_SEND_TIMEOUT = "2 seconds" -const toPersistenceError = (error: DatabaseError) => - new AuditLogPersistenceError({ message: error.message, cause: error }) +const toPersistenceError = (error: { readonly _tag: string; readonly message?: string }) => + new AuditLogPersistenceError({ message: error.message ?? error._tag, cause: error }) /** The credential-holder behind an audited action, as known at the call site. */ export interface AuditActorRef { @@ -88,25 +99,116 @@ export interface AuditLogListFilters { export interface AuditLogServiceApi { /** * Append one entry, durably: published to the audit events queue when the - * binding is present (the consumer performs the insert, retried by the - * queue), written straight to Postgres otherwise (tests, local dev, crons). - * Never fails: a mutation that succeeded must not 500 because its audit + * binding is present (the consumer performs the warehouse write, retried by + * the queue), written straight to the warehouse otherwise (local dev, crons). + * Never fails: an action that succeeded must not 500 because its audit * write did not — terminal failures are logged and swallowed. */ readonly record: (input: AuditLogRecordInput) => Effect.Effect readonly list: ( orgId: OrgId, filters: AuditLogListFilters, - ) => Effect.Effect, AuditLogPersistenceError> + ) => Effect.Effect, AuditLogPersistenceError> +} + +/** Build the queue event for one `record` call, stamping id and `occurredAt`. */ +const makeEvent = (input: AuditLogRecordInput, nowMs: number) => { + const resource = auditResourceFields(input.action, input.resourceId) + return new AuditLogEvent({ + orgId: input.orgId, + id: decodeAuditLogEntryIdSync(randomUUID()), + actorType: input.actor.type, + ...(input.actor.userId !== undefined ? { userId: input.actor.userId } : undefined), + ...(input.actor.apiKeyId !== undefined ? { apiKeyId: input.actor.apiKeyId } : undefined), + ...(input.actor.actorId !== undefined ? { actorId: input.actor.actorId } : undefined), + ...(input.actor.label !== undefined ? { actorLabel: input.actor.label } : undefined), + ...(input.affectedUserId !== undefined ? { affectedUserId: input.affectedUserId } : undefined), + source: input.source, + action: input.action, + outcome: input.outcome ?? "allowed", + ...(input.denialReason !== undefined ? { denialReason: input.denialReason } : undefined), + resourceType: resource.resourceType, + ...(resource.resourceId !== undefined ? { resourceId: resource.resourceId } : undefined), + ...(input.changes !== undefined ? { changes: input.changes } : undefined), + ...(input.metadata !== undefined ? { metadata: input.metadata } : undefined), + ...(input.requestId !== undefined ? { requestId: input.requestId } : undefined), + ...(input.originIp !== undefined ? { originIp: input.originIp } : undefined), + ...(input.originCountry !== undefined ? { originCountry: input.originCountry } : undefined), + occurredAtMs: nowMs, + }) +} + +/** Self-observability: refused attempts are the entries worth alerting on. */ +const logDenied = (event: AuditLogEvent) => + event.outcome === "denied" + ? Effect.logWarning("Audit: denied action").pipe( + Effect.annotateLogs({ + orgId: event.orgId, + action: event.action, + actorType: event.actorType, + denialReason: event.denialReason ?? "", + }), + ) + : Effect.void + +/** + * `record` never fails: swallow typed failures and defects — an action that + * succeeded must not 500 because its audit write did not — but let interrupts + * propagate so fiber teardown never triggers a stray write. + */ +const neverFail = (action: string) => (write: Effect.Effect) => + write.pipe( + Effect.catch((error) => Effect.logWarning("Audit log write failed", { action, cause: error })), + Effect.catchDefect((defect) => Effect.logWarning("Audit log write failed", { action, cause: defect })), + ) + +/** Which optional filters bind, and the parameter values behind them. */ +const listQueryInputs = (orgId: OrgId, filters: AuditLogListFilters) => { + const since = filters.sinceMs === undefined ? undefined : msToWarehouseDateTime64(filters.sinceMs) + const until = filters.untilMs === undefined ? undefined : msToWarehouseDateTime64(filters.untilMs) + const opts: CH.AuditLogEntriesOpts = { + actorType: filters.actorType !== undefined, + userId: filters.userId !== undefined, + apiKeyId: filters.apiKeyId !== undefined, + actorId: filters.actorId !== undefined, + affectedUserId: filters.affectedUserId !== undefined, + action: filters.action !== undefined, + outcome: filters.outcome !== undefined, + resourceType: filters.resourceType !== undefined, + resourceId: filters.resourceId !== undefined, + changedField: filters.changedField !== undefined, + requestId: filters.requestId !== undefined, + since: since !== undefined, + until: until !== undefined, + limit: filters.limit, + offset: filters.offset, + } + const values = { + orgId, + ...(filters.actorType !== undefined ? { actorType: filters.actorType } : undefined), + ...(filters.userId !== undefined ? { userId: filters.userId } : undefined), + ...(filters.apiKeyId !== undefined ? { apiKeyId: filters.apiKeyId } : undefined), + ...(filters.actorId !== undefined ? { actorId: filters.actorId } : undefined), + ...(filters.affectedUserId !== undefined ? { affectedUserId: filters.affectedUserId } : undefined), + ...(filters.action !== undefined ? { action: filters.action } : undefined), + ...(filters.outcome !== undefined ? { outcome: filters.outcome } : undefined), + ...(filters.resourceType !== undefined ? { resourceType: filters.resourceType } : undefined), + ...(filters.resourceId !== undefined ? { resourceId: filters.resourceId } : undefined), + ...(filters.changedField !== undefined ? { changedField: filters.changedField } : undefined), + ...(filters.requestId !== undefined ? { requestId: filters.requestId } : undefined), + ...(since !== undefined ? { since } : undefined), + ...(until !== undefined ? { until } : undefined), + } + return { opts, values } } export class AuditLogService extends Context.Service()( "@maple/api/services/AuditLogService", { make: Effect.gen(function* () { - const database = yield* Database - // Optional so PGlite tests and non-Worker runtimes fall back to direct - // writes without providing a WorkerEnvironment. + const warehouse = yield* WarehouseQueryService + // Optional so tests and non-Worker runtimes fall back to direct writes + // without providing a WorkerEnvironment. const workerEnv = yield* Effect.serviceOption(WorkerEnvironment) const queue = Option.match(workerEnv, { onNone: () => undefined, @@ -117,33 +219,36 @@ export class AuditLogService extends Context.Service + // One row through the managed ingest pipeline. `ingest` is pinned to + // Tinybird regardless of the org's read backend, which is the point: + // the audit log is Maple's record and never lands in a BYO warehouse. + const writeDirect = (event: AuditLogEvent) => Effect.gen(function* () { const now = yield* Clock.currentTimeMillis - yield* database.execute((db) => - db.insert(auditLogEntries).values(auditEventToInsert(event, now)).onConflictDoNothing(), - ) + yield* warehouse.ingest(systemTenant(event.orgId), AUDIT_LOG_DATASOURCE, [ + auditEventToRow(event, now), + ]) }) // Queue unavailability must not lose the entry: degrade to a direct // write before giving up. Only typed send failures land here — an - // interrupt must propagate, not spawn a Postgres insert mid-teardown. + // interrupt must propagate, not spawn a warehouse write mid-teardown. const fallbackToDirect = (event: AuditLogEvent, error: AuditQueueSendError) => Effect.logWarning("Audit queue send failed; writing directly", { cause: error }).pipe( - Effect.andThen(insertDirect(event)), + Effect.andThen(writeDirect(event)), ) const publish = (event: AuditLogEvent) => queue === undefined - ? insertDirect(event) + ? writeDirect(event) : Effect.tryPromise({ try: () => queue.send(encodeAuditLogEventSync(event)), catch: (cause) => new AuditQueueSendError({ message: "Audit queue send failed", cause }), }).pipe( // A Queues brown-out that stalls (rather than rejects) must not - // hang the mutation's response: 2s is far above a healthy send's - // latency yet bounds the worst case before the direct-write fallback. + // hang the response: 2s is far above a healthy send's latency + // yet bounds the worst case before the direct-write fallback. Effect.timeout(AUDIT_QUEUE_SEND_TIMEOUT), Effect.catchTag("TimeoutError", (error) => Effect.fail(new AuditQueueSendError({ message: "Audit queue send timed out", cause: error })), @@ -157,107 +262,34 @@ export class AuditLogService extends Context.Service - Effect.logWarning("Audit log write failed", { action: input.action, cause: error }), - ), - Effect.catchDefect((defect) => - Effect.logWarning("Audit log write failed", { action: input.action, cause: defect }), - ), - ) + const event = makeEvent(input, now) + yield* logDenied(event) + yield* publish(event).pipe(neverFail(input.action)) }) const list: AuditLogServiceApi["list"] = Effect.fn("AuditLogService.list")(function* ( orgId, filters, ) { - const conditions = [ - eq(auditLogEntries.orgId, orgId), - ...(filters.actorType !== undefined - ? [eq(auditLogEntries.actorType, filters.actorType)] - : []), - ...(filters.userId !== undefined ? [eq(auditLogEntries.userId, filters.userId)] : []), - ...(filters.apiKeyId !== undefined - ? [eq(auditLogEntries.apiKeyId, filters.apiKeyId)] - : []), - ...(filters.actorId !== undefined ? [eq(auditLogEntries.actorId, filters.actorId)] : []), - ...(filters.affectedUserId !== undefined - ? [eq(auditLogEntries.affectedUserId, filters.affectedUserId)] - : []), - ...(filters.action !== undefined ? [eq(auditLogEntries.action, filters.action)] : []), - ...(filters.outcome !== undefined ? [eq(auditLogEntries.outcome, filters.outcome)] : []), - ...(filters.resourceType !== undefined - ? [eq(auditLogEntries.resourceType, filters.resourceType)] - : []), - ...(filters.resourceId !== undefined - ? [eq(auditLogEntries.resourceId, filters.resourceId)] - : []), - ...(filters.changedField !== undefined - ? [arrayContains(auditLogEntries.changedFields, [filters.changedField])] - : []), - ...(filters.requestId !== undefined - ? [eq(auditLogEntries.requestId, filters.requestId)] - : []), - ...(filters.sinceMs !== undefined - ? [gte(auditLogEntries.occurredAt, msToDate(filters.sinceMs))] - : []), - ...(filters.untilMs !== undefined - ? [lte(auditLogEntries.occurredAt, msToDate(filters.untilMs))] - : []), - ] - return yield* database - .execute((db) => - db - .select() - .from(auditLogEntries) - .where(and(...conditions)) - .orderBy(desc(auditLogEntries.occurredAt), desc(auditLogEntries.id)) - .limit(filters.limit) - .offset(filters.offset), + const { opts, values } = listQueryInputs(orgId, filters) + const rows = yield* warehouse + .compiledQuery( + systemTenant(orgId), + CH.compile(CH.auditLogEntriesQuery(opts), values), + { profile: "list", context: "auditLog.list" }, ) .pipe(Effect.mapError(toPersistenceError)) + return yield* Effect.forEach(rows, (row) => + decodeStoredAuditLogEntry(row).pipe( + Effect.map((decoded) => storedRowToEntry(orgId, decoded)), + Effect.mapError((error) => + new AuditLogPersistenceError({ + message: "Stored audit entry failed to decode", + cause: error, + }), + ), + ), + ) }) return { record, list } @@ -265,22 +297,73 @@ export class AuditLogService extends Context.Service = [] + const record: AuditLogServiceApi["record"] = (input) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const event = makeEvent(input, now) + yield* logDenied(event) + entries.push(auditEventToEntry(event, now)) + }) + const list: AuditLogServiceApi["list"] = (orgId, filters) => + Effect.succeed( + entries + .filter( + (entry) => + entry.orgId === orgId && + (filters.actorType === undefined || entry.actorType === filters.actorType) && + (filters.userId === undefined || entry.userId === filters.userId) && + (filters.apiKeyId === undefined || entry.apiKeyId === filters.apiKeyId) && + (filters.actorId === undefined || entry.actorId === filters.actorId) && + (filters.affectedUserId === undefined || + entry.affectedUserId === filters.affectedUserId) && + (filters.action === undefined || entry.action === filters.action) && + (filters.outcome === undefined || entry.outcome === filters.outcome) && + (filters.resourceType === undefined || entry.resourceType === filters.resourceType) && + (filters.resourceId === undefined || entry.resourceId === filters.resourceId) && + (filters.changedField === undefined || + (entry.changedFields?.includes(filters.changedField) ?? false)) && + (filters.requestId === undefined || entry.requestId === filters.requestId) && + (filters.sinceMs === undefined || entry.occurredAt.getTime() >= filters.sinceMs) && + (filters.untilMs === undefined || entry.occurredAt.getTime() <= filters.untilMs), + ) + .sort( + (a, b) => + b.occurredAt.getTime() - a.occurredAt.getTime() || (b.id < a.id ? -1 : b.id > a.id ? 1 : 0), + ) + .slice(filters.offset, filters.offset + filters.limit), + ) + return { record, list } } /** Request forensics for an audit entry, read off the Cloudflare request headers. */ -const requestContext = Effect.gen(function* () { +export const httpRequestForensics = (request: HttpServerRequest.HttpServerRequest) => ({ + ...(request.headers["cf-ray"] !== undefined ? { requestId: request.headers["cf-ray"] } : undefined), + ...(request.headers["cf-connecting-ip"] !== undefined + ? { originIp: request.headers["cf-connecting-ip"] } + : undefined), + ...(request.headers["cf-ipcountry"] !== undefined + ? { originCountry: request.headers["cf-ipcountry"] } + : undefined), +}) + +/** Forensics for the current request, or nothing outside an HTTP request. */ +export const currentRequestForensics = Effect.gen(function* () { const request = yield* Effect.serviceOption(HttpServerRequest.HttpServerRequest) return Option.match(request, { onNone: () => ({}), - onSome: (req) => ({ - ...(req.headers["cf-ray"] !== undefined ? { requestId: req.headers["cf-ray"] } : undefined), - ...(req.headers["cf-connecting-ip"] !== undefined - ? { originIp: req.headers["cf-connecting-ip"] } - : undefined), - ...(req.headers["cf-ipcountry"] !== undefined - ? { originCountry: req.headers["cf-ipcountry"] } - : undefined), - }), + onSome: httpRequestForensics, }) }) @@ -304,7 +387,7 @@ export const recordHttpAudit = ( const audit = yield* AuditLogService const tenant = yield* CurrentTenant.Context const info = yield* CurrentAuditActor - const context = yield* requestContext + const context = yield* currentRequestForensics // No reference means the request bypassed every auth middleware (internal // tokens, tests). Attribute to the tenant's user rather than inventing a // credential, but do not claim a surface the request may not have used. diff --git a/apps/api/src/services/audit/audit-access.ts b/apps/api/src/services/audit/audit-access.ts new file mode 100644 index 000000000..aef9b0f15 --- /dev/null +++ b/apps/api/src/services/audit/audit-access.ts @@ -0,0 +1,223 @@ +import { Context, Effect } from "effect" +import type { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import type { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" +import { AuditedRead, type AuditLogSource } from "@maple/domain/http" +import type { ActorId, OrgId, UserId } from "@maple/domain/primitives" +import type { McpToolSurface } from "@/mcp/dispatcher" +import type { AuditActorInfo } from "@/services/auth/audit-actor" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import type { TenantContext } from "@/services/auth/tenant-context" +import { + type AuditActorRef, + AuditLogService, + type AuditLogServiceApi, + currentRequestForensics, + httpRequestForensics, +} from "./AuditLogService" + +/** + * Access auditing: who read what. HIPAA's audit-control standard covers every + * access to protected data, not only changes, so three read surfaces record + * entries alongside the mutation trail — + * + * - HTTP endpoints annotated `AuditedRead` (telemetry and session replays), + * wrapped by the auth layers via {@link withAuditedRead}; + * - every MCP tool invocation, from the executor via {@link recordMcpToolAudit}; + * - every raw SQL statement, via {@link recordRawSqlAudit}. + */ + +/** Bound on stored request/parameter snapshots — enough to see what was asked for. */ +const MAX_SNAPSHOT_CHARS = 2_000 + +/** A JSON rendering of `value` capped at {@link MAX_SNAPSHOT_CHARS}. */ +export const snapshot = (value: unknown): string => { + const text = typeof value === "string" ? value : (JSON.stringify(value) ?? "null") + return text.length > MAX_SNAPSHOT_CHARS ? `${text.slice(0, MAX_SNAPSHOT_CHARS)}…` : text +} + +export interface AuditAttribution { + readonly actor: AuditActorRef + readonly source: AuditLogSource +} + +/** + * Attribute an action performed under `tenant`. The credential and surface + * come from the auth layer's `CurrentAuditActor` when one set it; an agent + * tenant (pinned `actorId`) is recorded as the agent acting on the user's + * behalf; nothing set means a dashboard session, never a guessed credential. + */ +/** The tenant facts attribution needs; both `TenantContext` and `TenantSchema` satisfy it. */ +export interface AuditTenant { + readonly orgId: OrgId + readonly userId: UserId + readonly actorId?: ActorId | undefined + readonly mcpClientName?: string | undefined +} + +export const auditAttribution = (tenant: AuditTenant, info: AuditActorInfo | undefined): AuditAttribution => { + if (info?.type === "system") return { actor: { type: "system" }, source: "system" } + if (tenant.actorId !== undefined) { + return { + actor: { + type: "agent", + actorId: tenant.actorId, + userId: tenant.userId, + ...(tenant.mcpClientName !== undefined ? { label: tenant.mcpClientName } : undefined), + }, + source: info?.source ?? "mcp", + } + } + return { + actor: { + type: info?.type ?? "user", + userId: tenant.userId, + ...(info?.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), + }, + source: info?.source ?? "dashboard", + } +} + +export interface AuditedReadSubject extends AuditAttribution { + readonly orgId: OrgId +} + +/** + * Wrap an authenticated endpoint response so that, when the endpoint is + * annotated `AuditedRead`, the call is recorded once it completes — with the + * HTTP status, the route, the request path, and (for POST searches) a bounded + * snapshot of the body that says what was queried. A typed failure still + * records an attempt; an interrupt records nothing. + */ +export const withAuditedRead = + ( + audit: AuditLogServiceApi, + request: HttpServerRequest.HttpServerRequest, + options: { readonly endpoint: HttpApiEndpoint.Top; readonly group: HttpApiGroup.Top }, + subject: AuditedReadSubject, + ) => + ( + httpEffect: Effect.Effect, + ): Effect.Effect => { + const action = Context.get(options.endpoint.annotations, AuditedRead) + if (action === undefined) return httpEffect + const record = (status: number) => + Effect.gen(function* () { + // The handler already consumed (and cached) the body, so this is a + // read of the same text, never a second parse of the stream. + const body = + request.method === "GET" || request.method === "HEAD" + ? undefined + : yield* request.text.pipe(Effect.option) + yield* audit.record({ + orgId: subject.orgId, + actor: subject.actor, + source: subject.source, + action, + metadata: { + endpoint: `${options.group.identifier}.${options.endpoint.name}`, + method: request.method, + path: request.url, + status, + ...(body !== undefined && body._tag === "Some" && body.value !== "" + ? { body: snapshot(body.value) } + : undefined), + }, + ...httpRequestForensics(request), + }) + }) + return httpEffect.pipe( + Effect.tap((response) => record(response.status)), + // A rejected read (bad parameters, not found) is still an attempt; + // 0 says the status was never produced. + Effect.tapError(() => record(0)), + ) + } + +export interface McpToolAuditInput { + readonly tenant: TenantContext + readonly name: string + readonly input: unknown + readonly surface: McpToolSurface + readonly isError: boolean +} + +/** + * One `mcp_tool.called` entry per tool invocation, whichever surface drove it. + * Workflow passes and internal RPC run under Maple's own tenant, so they are + * `system`; the public transport and the chat attribute through the tenant. + */ +export const recordMcpToolAudit = (input: McpToolAuditInput) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const info = yield* CurrentAuditActor + const forensics = yield* currentRequestForensics + const attribution = + input.surface === "workflow" || input.surface === "rpc" + ? { actor: { type: "system" as const, label: input.surface }, source: "system" as const } + : auditAttribution(input.tenant, info) + yield* audit.record({ + orgId: input.tenant.orgId, + ...attribution, + action: "mcp_tool.called", + metadata: { + tool: input.name, + surface: input.surface, + is_error: input.isError, + params: snapshot(input.input), + }, + ...forensics, + }) + }) + +export type RawSqlAuditResult = + | { readonly _tag: "rows"; readonly rowCount: number } + /** The safety pass refused the statement before it ran. */ + | { readonly _tag: "rejected"; readonly reason: string } + /** The warehouse refused or failed the statement. */ + | { readonly _tag: "failed"; readonly error: string } + +export interface RawSqlAuditInput { + readonly tenant: AuditTenant + readonly sql: string + /** The executor context label: `mcp.run_sql`, `rawSql`, … */ + readonly context: string + readonly startTime: string + readonly endTime: string + readonly result: RawSqlAuditResult +} + +/** + * One `telemetry.sql_executed` entry per raw SQL statement — the statement + * itself, the window it ran over, and how it ended. A statement the safety + * pass refused is a `denied` entry: an attempt to read outside the guardrails + * is exactly what an auditor asks about. + */ +export const recordRawSqlAudit = (input: RawSqlAuditInput) => + Effect.gen(function* () { + const audit = yield* AuditLogService + const info = yield* CurrentAuditActor + const forensics = yield* currentRequestForensics + const { actor, source } = auditAttribution(input.tenant, info) + yield* audit.record({ + orgId: input.tenant.orgId, + actor, + source, + action: "telemetry.sql_executed", + ...(input.result._tag === "rejected" + ? { outcome: "denied", denialReason: input.result.reason } + : undefined), + metadata: { + sql: snapshot(input.sql), + context: input.context, + start_time: input.startTime, + end_time: input.endTime, + ...(input.result._tag === "rows" ? { row_count: input.result.rowCount } : undefined), + ...(input.result._tag === "failed" ? { error: input.result.error } : undefined), + }, + ...forensics, + }) + }) + +/** A one-line description of a typed failure for the audit metadata. */ +export const describeFailure = (error: { readonly _tag: string; readonly message?: string }): string => + error.message ?? error._tag diff --git a/apps/api/src/services/audit/audit-event.ts b/apps/api/src/services/audit/audit-event.ts index 30e79b5c5..7c935e03e 100644 --- a/apps/api/src/services/audit/audit-event.ts +++ b/apps/api/src/services/audit/audit-event.ts @@ -1,12 +1,17 @@ -import { AuditActorType, AuditChanges, AuditLogSource, AuditOutcome } from "@maple/domain/http" +import { + AuditActorType, + AuditChanges, + AuditLogSource, + AuditOutcome, +} from "@maple/domain/http" import { ActorId, ApiKeyId, AuditLogEntryId, OrgId, UserId } from "@maple/domain/primitives" -import type { AuditLogEntryInsert } from "@maple/db" -import { Schema } from "effect" -import { msToDate } from "@/platform/time" +import type { AuditLogRow } from "@maple/domain/tinybird" +import { Schema, SchemaTransformation } from "effect" +import { msToDate, msToWarehouseDateTime64 } from "@/platform/time" /** * The serialized audit event as it travels the audit queue. `occurredAtMs` is - * stamped by the producer; `recordedAt` exists only on the table row, stamped + * stamped by the producer; `recordedAt` exists only on the stored row, stamped * by whichever writer performs the insert. */ export class AuditLogEvent extends Schema.Class("AuditLogEvent")({ @@ -35,8 +40,64 @@ export class AuditLogEvent extends Schema.Class("AuditLogEvent")( export const decodeAuditLogEvent = Schema.decodeUnknownEffect(AuditLogEvent) export const encodeAuditLogEventSync = Schema.encodeSync(AuditLogEvent) -/** Lower a queue event to its table row; `recordedAtMs` is the insert time. */ -export const auditEventToInsert = (event: AuditLogEvent, recordedAtMs: number): AuditLogEntryInsert => ({ +/** + * One stored audit entry, as the service hands it to readers. Absent values are + * `null` here and `''` in the warehouse row; the two lowering functions below + * are the only places that mapping lives. + */ +export interface AuditLogEntry { + readonly orgId: OrgId + readonly id: AuditLogEntryId + readonly actorType: AuditActorType + readonly userId: UserId | null + readonly apiKeyId: ApiKeyId | null + readonly actorId: ActorId | null + readonly actorLabel: string | null + readonly affectedUserId: UserId | null + readonly source: AuditLogSource + readonly action: string + readonly outcome: AuditOutcome + readonly denialReason: string | null + readonly resourceType: string | null + readonly resourceId: string | null + readonly changedFields: ReadonlyArray | null + readonly changes: AuditChanges | null + readonly metadata: Record | null + readonly requestId: string | null + readonly originIp: string | null + readonly originCountry: string | null + readonly occurredAt: Date + readonly recordedAt: Date +} + +/** Lower a queue event to its `audit_log` warehouse row; `recordedAtMs` is the write time. */ +export const auditEventToRow = (event: AuditLogEvent, recordedAtMs: number): AuditLogRow => ({ + OrgId: event.orgId, + Id: event.id, + OccurredAt: msToWarehouseDateTime64(event.occurredAtMs), + RecordedAt: msToWarehouseDateTime64(recordedAtMs), + ActorType: event.actorType, + UserId: event.userId ?? "", + ApiKeyId: event.apiKeyId ?? "", + ActorId: event.actorId ?? "", + ActorLabel: event.actorLabel ?? "", + AffectedUserId: event.affectedUserId ?? "", + Source: event.source, + Action: event.action, + Outcome: event.outcome, + DenialReason: event.denialReason ?? "", + ResourceType: event.resourceType ?? "", + ResourceId: event.resourceId ?? "", + ChangedFields: event.changes === undefined ? [] : [...event.changes.fields], + Changes: event.changes === undefined ? "" : JSON.stringify(event.changes), + Metadata: event.metadata === undefined ? "" : JSON.stringify(event.metadata), + RequestId: event.requestId ?? "", + OriginIp: event.originIp ?? "", + OriginCountry: event.originCountry ?? "", +}) + +/** The entry a reader would get back for `event` — what the in-memory layer stores. */ +export const auditEventToEntry = (event: AuditLogEvent, recordedAtMs: number): AuditLogEntry => ({ orgId: event.orgId, id: event.id, actorType: event.actorType, @@ -52,11 +113,83 @@ export const auditEventToInsert = (event: AuditLogEvent, recordedAtMs: number): resourceType: event.resourceType ?? null, resourceId: event.resourceId ?? null, changedFields: event.changes === undefined ? null : [...event.changes.fields], - changesJson: event.changes ?? null, - metadataJson: event.metadata ?? null, + changes: event.changes ?? null, + metadata: event.metadata ?? null, requestId: event.requestId ?? null, originIp: event.originIp ?? null, originCountry: event.originCountry ?? null, occurredAt: msToDate(event.occurredAtMs), recordedAt: msToDate(recordedAtMs), }) + +const JsonRecord = Schema.Record(Schema.String, Schema.Unknown) + +/** `''` in the warehouse row is "absent"; everything else decodes through `schema`. */ +const emptyAsNull = >(schema: S) => + Schema.String.pipe( + Schema.decodeTo( + Schema.NullOr(Schema.String), + SchemaTransformation.transform({ + decode: (value: string) => (value === "" ? null : value), + encode: (value: string | null) => value ?? "", + }), + ), + Schema.decodeTo(Schema.NullOr(schema)), + ) + +const nullableText = emptyAsNull(Schema.String) + +/** JSON document columns: `''` when absent, otherwise a JSON string of `schema`. */ +const jsonDocument = (schema: S) => emptyAsNull(Schema.fromJsonString(schema)) + +/** `YYYY-MM-DD HH:mm:ss.SSS` (UTC, as the warehouse emits DateTime64) ⇄ `Date`. */ +const warehouseDateTime = Schema.String.pipe( + Schema.decodeTo( + Schema.Date, + SchemaTransformation.transform({ + decode: (value: string) => new Date(`${value.replace(" ", "T")}Z`), + encode: (value: Date) => msToWarehouseDateTime64(value.getTime()), + }), + ), +) + +/** + * A listed row exactly as the warehouse returns it, with the `''`-means-absent + * convention decoded back to `null` and the JSON document columns parsed. + */ +export const StoredAuditLogEntry = Schema.Struct({ + id: AuditLogEntryId, + occurredAt: warehouseDateTime, + recordedAt: warehouseDateTime, + actorType: AuditActorType, + userId: emptyAsNull(UserId), + apiKeyId: emptyAsNull(ApiKeyId), + actorId: emptyAsNull(ActorId), + actorLabel: nullableText, + affectedUserId: emptyAsNull(UserId), + source: AuditLogSource, + action: Schema.String, + outcome: AuditOutcome, + denialReason: nullableText, + resourceType: nullableText, + resourceId: nullableText, + changedFields: Schema.Array(Schema.String), + changes: jsonDocument(AuditChanges), + metadata: jsonDocument(JsonRecord), + requestId: nullableText, + originIp: nullableText, + originCountry: nullableText, +}) + +export const decodeStoredAuditLogEntry = Schema.decodeUnknownEffect(StoredAuditLogEntry) + +/** A decoded warehouse row as an `AuditLogEntry`. */ +export const storedRowToEntry = ( + orgId: OrgId, + row: Schema.Schema.Type, +): AuditLogEntry => ({ + orgId, + ...row, + // An entry with no diff has no changed fields either; the row stores `[]`. + changedFields: row.changes === null ? null : row.changedFields, +}) diff --git a/apps/api/src/services/auth/ApiAuthorizationLayer.ts b/apps/api/src/services/auth/ApiAuthorizationLayer.ts index 6c57bf3ac..01ca47857 100644 --- a/apps/api/src/services/auth/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationLayer.ts @@ -7,6 +7,7 @@ import { annotateAuthSpan } from "@/services/auth/auth-span" import { CurrentAuditActor } from "@/services/auth/audit-actor" import { AuditLogService } from "@/services/audit/AuditLogService" import { recordApiDenial } from "@/services/auth/audit-denial" +import { withAuditedRead } from "@/services/audit/audit-access" import { Env } from "@/platform/Env" const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) @@ -29,7 +30,7 @@ export const ApiAuthorizationLayer = Layer.effect( const resolveTenant = makeResolveTenant(env) return CurrentTenant.Authorization.of({ - bearer: (httpEffect) => + bearer: (httpEffect, options) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest @@ -87,6 +88,11 @@ export const ApiAuthorizationLayer = Layer.effect( apiKeyId: resolved.keyId, source: "api", }), + withAuditedRead(audit, request, options, { + orgId: resolved.orgId, + actor: { type: "api_key", userId: resolved.userId, apiKeyId: resolved.keyId }, + source: "api", + }), ) } @@ -95,6 +101,11 @@ export const ApiAuthorizationLayer = Layer.effect( return yield* httpEffect.pipe( Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), + withAuditedRead(audit, request, options, { + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: "dashboard", + }), ) }), }) diff --git a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts index b80b33123..67d8a71ad 100644 --- a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts @@ -18,6 +18,7 @@ import { annotateAuthSpan } from "@/services/auth/auth-span" import { CurrentAuditActor } from "@/services/auth/audit-actor" import { AuditLogService } from "@/services/audit/AuditLogService" import { recordApiDenial } from "@/services/auth/audit-denial" +import { withAuditedRead } from "@/services/audit/audit-access" import { Env } from "@/platform/Env" import { API_V2_RATE_LIMIT_PERIOD_SECONDS, @@ -87,7 +88,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( ) return AuthorizationV2.of({ - bearer: (httpEffect) => + bearer: (httpEffect, options) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest @@ -204,6 +205,12 @@ export const ApiAuthorizationV2Layer = Layer.effect( apiKeyId: resolved.keyId, source: "api", }), + // Telemetry and replay reads are recorded (see `AuditedRead`). + withAuditedRead(audit, request, options, { + orgId: resolved.orgId, + actor: { type: "api_key", userId: resolved.userId, apiKeyId: resolved.keyId }, + source: "api", + }), ) } @@ -216,6 +223,11 @@ export const ApiAuthorizationV2Layer = Layer.effect( return yield* httpEffect.pipe( Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), + withAuditedRead(audit, request, options, { + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: "dashboard", + }), ) }), }) diff --git a/apps/api/src/services/auth/SessionAuthorizationLayer.ts b/apps/api/src/services/auth/SessionAuthorizationLayer.ts index 6c2df1fce..a2e77d3f2 100644 --- a/apps/api/src/services/auth/SessionAuthorizationLayer.ts +++ b/apps/api/src/services/auth/SessionAuthorizationLayer.ts @@ -5,6 +5,8 @@ import { Effect, Layer } from "effect" import { makeResolveTenant } from "./AuthService" import { annotateAuthSpan } from "@/services/auth/auth-span" import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { AuditLogService } from "@/services/audit/AuditLogService" +import { withAuditedRead } from "@/services/audit/audit-access" import { Env } from "@/platform/Env" const getBearerToken = (headers: Record): string | undefined => { @@ -32,10 +34,11 @@ export const SessionAuthorizationLayer = Layer.effect( CurrentTenant.SessionAuthorization, Effect.gen(function* () { const env = yield* Env + const audit = yield* AuditLogService const resolveTenant = makeResolveTenant(env) return CurrentTenant.SessionAuthorization.of({ - bearer: (httpEffect) => + bearer: (httpEffect, options) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest @@ -48,9 +51,16 @@ export const SessionAuthorizationLayer = Layer.effect( const tenant = yield* resolveTenant(request.headers) yield* annotateAuthSpan("session", { orgId: tenant.orgId, userId: tenant.userId }) + const actor = { type: "user", source: "dashboard" } as const return yield* httpEffect.pipe( Effect.provideService(CurrentTenant.Context, new CurrentTenant.TenantSchema(tenant)), - Effect.provideService(CurrentAuditActor, { type: "user", source: "dashboard" }), + Effect.provideService(CurrentAuditActor, actor), + // Telemetry and replay reads are recorded (see `AuditedRead`). + withAuditedRead(audit, request, options, { + orgId: tenant.orgId, + actor: { type: "user", userId: tenant.userId }, + source: actor.source, + }), ) }), }) diff --git a/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts b/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts index e2002a232..2097be7f2 100644 --- a/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueReadModelsService.test.ts @@ -68,7 +68,7 @@ const makeLayer = (contexts: Array) => { const database = createTestDb(createdDbs).layer const actors = ErrorActorsService.layer.pipe(Layer.provide(database)) const workflow = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(AuditLogService.layer), + Layer.provide(AuditLogService.layerMemory), Layer.provide(database), Layer.provide(actors), ) diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts index c9548dff1..c8435e329 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts @@ -48,7 +48,7 @@ afterEach(() => cleanupTestDbs(createdDbs)) const makeLayer = () => { const database = createTestDb(createdDbs).layer const actors = ErrorActorsService.layer.pipe(Layer.provide(database)) - const audit = AuditLogService.layer.pipe(Layer.provide(database)) + const audit = AuditLogService.layerMemory const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(database, actors, audit))) return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(database)) } @@ -99,7 +99,9 @@ const makeFaultyLayer = (failTable: unknown) => { }), ).pipe(Layer.provide(database)) const actors = ErrorActorsService.layer.pipe(Layer.provide(faulty)) - const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(faulty, actors))) + const workflow = databaseAndActorsOnly.pipe( + Layer.provide(Layer.mergeAll(faulty, actors, AuditLogService.layerMemory)), + ) return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(faulty)) } diff --git a/apps/api/src/services/errors/ErrorsService.test.ts b/apps/api/src/services/errors/ErrorsService.test.ts index bb816e014..f5855959f 100644 --- a/apps/api/src/services/errors/ErrorsService.test.ts +++ b/apps/api/src/services/errors/ErrorsService.test.ts @@ -210,7 +210,7 @@ const makeErrorsLayer = ( const databaseLive = testDb.layer const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const errorIssueWorkflowLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(AuditLogService.layer), + Layer.provide(AuditLogService.layerMemory), Layer.provide(databaseLive), Layer.provide(errorActorsLive), ) @@ -301,7 +301,7 @@ const makeGatingLayer = (opts: { const databaseLive = testDb.layer const errorActorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const errorIssueWorkflowLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(AuditLogService.layer), + Layer.provide(AuditLogService.layerMemory), Layer.provide(databaseLive), Layer.provide(errorActorsLive), ) diff --git a/apps/api/src/services/errors/IssueFixVerificationService.test.ts b/apps/api/src/services/errors/IssueFixVerificationService.test.ts index 5691206ab..c970a5aa0 100644 --- a/apps/api/src/services/errors/IssueFixVerificationService.test.ts +++ b/apps/api/src/services/errors/IssueFixVerificationService.test.ts @@ -69,7 +69,7 @@ const makeLayer = ( const envLive = Env.layer.pipe(Layer.provide(testConfig())) const actorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const workflowLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide(AuditLogService.layer), + Layer.provide(AuditLogService.layerMemory), Layer.provide(databaseLive), Layer.provide(actorsLive), ) diff --git a/apps/api/src/vcs-sync-runtime.ts b/apps/api/src/vcs-sync-runtime.ts index ac42877d9..114a04315 100644 --- a/apps/api/src/vcs-sync-runtime.ts +++ b/apps/api/src/vcs-sync-runtime.ts @@ -3,7 +3,12 @@ import * as MapleCloudflareSDK from "@maple-dev/effect-sdk/cloudflare" import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" import { WorkerConfigProviderLayer, WorkerEnvironment } from "@maple/effect-cloudflare" import { Cause, Effect, Layer, Option } from "effect" +import { EdgeCacheService } from "@maple/cache" +import { CacheBackendLive } from "@/platform/CacheBackendLive" import { layerPg } from "@/platform/DatabasePgLive" +import { TinybirdOrgTokenService } from "@/services/integrations/TinybirdOrgTokenService" +import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" +import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { AuditLogService } from "@/services/audit/AuditLogService" import { Env } from "@/platform/Env" import { GithubAppClient } from "./services/integrations/vcs/vendor/github/GithubAppClient" @@ -56,10 +61,21 @@ export const buildVcsSyncLayer = (_env: Record) => { // the scheduled producer below never sees a PR event — so it is built here // rather than in `Base`, keeping the cron layer as light as it was. const ErrorActorsServiceLive = ErrorActorsService.layer.pipe(Layer.provide(Base)) + // Issue events from a PR webhook are audited, and audit entries are warehouse + // rows — so the consumer carries the (Tinybird-pinned) ingest path as well. + const EdgeCacheServiceLive = EdgeCacheService.layer.pipe(Layer.provide(CacheBackendLive)) + const OrgClickHouseSettingsLive = OrgClickHouseSettingsService.layer.pipe( + Layer.provide(Layer.mergeAll(Base, EdgeCacheServiceLive)), + ) + const TinybirdOrgTokenLive = TinybirdOrgTokenService.layer.pipe(Layer.provide(EnvLive)) + const WarehouseQueryServiceLive = WarehouseQueryService.layer.pipe( + Layer.provide(Layer.mergeAll(EnvLive, OrgClickHouseSettingsLive, TinybirdOrgTokenLive)), + ) + const AuditLogServiceLive = AuditLogService.layer.pipe( + Layer.provide(Layer.mergeAll(WarehouseQueryServiceLive, WorkerEnvironment.layer)), + ) const ErrorIssueWorkflowServiceLive = ErrorIssueWorkflowService.layer.pipe( - Layer.provide( - Layer.mergeAll(Base, ErrorActorsServiceLive, AuditLogService.layer.pipe(Layer.provide(Base))), - ), + Layer.provide(Layer.mergeAll(Base, ErrorActorsServiceLive, AuditLogServiceLive)), ) const IssueFixVerificationServiceLive = IssueFixVerificationService.layer.pipe( Layer.provide( diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 74351365b..7a97abeec 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -147,4 +147,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "faf78f67abd5901351ce6632cee59f22fadb3c1f7eb9b195dc2f9702d4c9c9bd", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + version: 15, + fingerprint: "9dcc61f3d2522826", + digest: "9dcc61f3d2522826070ac2063c777d34ec4b1f4492d4ac6663729c21c29a4495", + manifestDigest: "f7f9bf825e84924e9d6e0c86712eba5db249db9b8ceb07010d6926ad736bc5e7", + projectRevision: "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index af3f256d7..de7a1ff68 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 14 as const +export const LOCAL_SCHEMA_VERSION = 15 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 93ae4d3f9..4dd88fb38 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -50,6 +50,7 @@ import { v10ToV11ProductEventsModule } from "./local-store-migrations/v10-to-v11 import { v11ToV12ServiceMapEdgeQuantilesModule } from "./local-store-migrations/v11-to-v12-service-map-edge-quantiles" import { v12ToV13ServiceOperationsDiscriminatorsModule } from "./local-store-migrations/v12-to-v13-service-operations-discriminators" import { v13ToV14AiTraceIndexModule } from "./local-store-migrations/v13-to-v14-ai-trace-index" +import { v14ToV15AuditLogModule } from "./local-store-migrations/v14-to-v15-audit-log" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -121,6 +122,7 @@ export const localStoreMigrations: ReadonlyArray = v11ToV12ServiceMapEdgeQuantilesModule, v12ToV13ServiceOperationsDiscriminatorsModule, v13ToV14AiTraceIndexModule, + v14ToV15AuditLogModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v14-to-v15-audit-log.ts b/apps/cli/src/server/local-store-migrations/v14-to-v15-audit-log.ts new file mode 100644 index 000000000..89abe340d --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v14-to-v15-audit-log.ts @@ -0,0 +1,150 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { + LOCAL_SCHEMA_V14, + LOCAL_SCHEMA_V14_MANIFEST, + LOCAL_SCHEMA_V14_SQL, + LOCAL_SCHEMA_V15, + LOCAL_SCHEMA_V15_MANIFEST, + LOCAL_SCHEMA_V15_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0014-to-0015-audit-log" as const + +const V14ToV15StateCodec = makeRawRowsState(MODULE_ID) + +type V14ToV15State = typeof V14ToV15StateCodec.schema.Type +type V14ToV15Progress = InstalledProgress + +const decodeState = V14ToV15StateCodec.decode +const decodeProgress = decodeInstalledProgress + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V14_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V14_SQL, bootstrapSchema: false }, + ) + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V14ToV15State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Purely additive: v15 introduces the `audit_log` table (ClickHouse migration + * 0025) and touches nothing else. Bootstrapping the v15 DDL over the cloned v14 + * store creates it through `CREATE TABLE IF NOT EXISTS`; every existing table + * and view is left as it is. Local mode has no authenticated actors, so the + * table starts and stays empty here — it exists so the local schema keeps + * mirroring the deployed one. + */ +const apply = async (context: MigrationModuleContext): Promise => + context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V15_SQL, + bootstrapSchema: true, + }) + +const verify = async ( + context: MigrationModuleContext, + state: V14ToV15State, + _progress: V14ToV15Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V15_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v14 -> v15 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V15_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v14-store", + description: "Clone the stopped v14 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "create-audit-log", + description: "Create the empty audit_log table by bootstrapping the v15 schema", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v15-schema", + description: "Verify the v15 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v14 store is cloned byte-for-byte before the new table is created.", + }, + { + name: "audit_log", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "Created empty; no existing table is read, rewritten, or dropped.", + }, +] + +export const v14ToV15AuditLogModule: LocalStoreMigrationModule = { + id: MODULE_ID, + moduleVersion: 1, + description: "Add the audit_log table", + from: LOCAL_SCHEMA_V14, + to: LOCAL_SCHEMA_V15, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 57734100e..6558cb153 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -13,6 +13,7 @@ import schemaV11Sql from "./schema/local-schema-v11.sql" with { type: "text" } import schemaV12Sql from "./schema/local-schema-v12.sql" with { type: "text" } import schemaV13Sql from "./schema/local-schema-v13.sql" with { type: "text" } import schemaV14Sql from "./schema/local-schema-v14.sql" with { type: "text" } +import schemaV15Sql from "./schema/local-schema-v15.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -36,7 +37,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a" + "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -71,6 +72,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV12Sql, schemaV13Sql, schemaV14Sql, + schemaV15Sql, ] export interface LocalSchemaSnapshot { @@ -123,6 +125,8 @@ export const LOCAL_SCHEMA_V13_SQL = snapshotAt(13).sql export const LOCAL_SCHEMA_V13_MANIFEST = snapshotAt(13).manifest export const LOCAL_SCHEMA_V14_SQL = snapshotAt(14).sql export const LOCAL_SCHEMA_V14_MANIFEST = snapshotAt(14).manifest +export const LOCAL_SCHEMA_V15_SQL = snapshotAt(15).sql +export const LOCAL_SCHEMA_V15_MANIFEST = snapshotAt(15).manifest export interface LocalSchemaIdentity { readonly version: number @@ -167,6 +171,7 @@ export const LOCAL_SCHEMA_V11 = identityAt(11) export const LOCAL_SCHEMA_V12 = identityAt(12) export const LOCAL_SCHEMA_V13 = identityAt(13) export const LOCAL_SCHEMA_V14 = identityAt(14) +export const LOCAL_SCHEMA_V15 = identityAt(15) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-schema-v15.sql b/apps/cli/src/server/schema/local-schema-v15.sql new file mode 100644 index 000000000..26586d01c --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v15.sql @@ -0,0 +1,1936 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748 +-- localSchemaVersion: 15 + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 82e116213..26586d01c 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. -- projectRevision: 6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748 --- localSchemaVersion: 14 +-- localSchemaVersion: 15 CREATE TABLE IF NOT EXISTS ai_trace_index ( OrgId LowCardinality(String), diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 41b3eaead..84f3476d1 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -29,6 +29,7 @@ import { LOCAL_SCHEMA_V13, LOCAL_SCHEMA_V13_MANIFEST, LOCAL_SCHEMA_V14, + LOCAL_SCHEMA_V15, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -76,16 +77,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v14 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("c46a599e1bfe417c") - expect(SCHEMA_DIGEST).toBe("c46a599e1bfe417c1e6f50d123779c6ca9c5f5f375ef9c6fe329c8a9676e3b5b") + it("matches the generated v15 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("9dcc61f3d2522826") + expect(SCHEMA_DIGEST).toBe("9dcc61f3d2522826070ac2063c777d34ec4b1f4492d4ac6663729c21c29a4495") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(14) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V14) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(15) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V15) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -153,7 +154,7 @@ describe("current local schema identity", () => { // bodies, so their object set is identical to v5 and the manifest digest // differs solely through those definitions. v9 removes `error_spans` and // its view; v11 replaces `web_events` with `product_events` and adds - // `identity_links`. Asserted as an exact set difference rather than a + // `identity_links`; v15 adds `audit_log`. Asserted as an exact set difference rather than a // relaxed check, so a future edge still cannot add or drop an object // unnoticed. expect([...v5Names].filter((name) => !currentNames.has(name))).toEqual([ @@ -165,6 +166,8 @@ describe("current local schema identity", () => { expect([...currentNames].filter((name) => !v5Names.has(name))).toEqual([ "ai_trace_index", "ai_trace_index_mv", + // v15: the org audit trail, empty in local mode (no authenticated actors). + "audit_log", "identity_links", "identity_links_mv", "product_events", @@ -253,7 +256,7 @@ describe("current local schema identity", () => { // v12 replaces two view bodies and v13 adds columns to two rollups; neither // adds an object. v14 is exactly the GenAI span index and its view, created - // empty and filled forward. + // empty and filled forward; v15 is exactly the audit trail, created empty. const v12Names = new Set(LOCAL_SCHEMA_V12_MANIFEST.objects.map((object) => object.name)) const v13Names = new Set(LOCAL_SCHEMA_V13_MANIFEST.objects.map((object) => object.name)) const currentSchemaNames = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) @@ -263,6 +266,7 @@ describe("current local schema identity", () => { expect([...currentSchemaNames].filter((name) => !v13Names.has(name))).toEqual([ "ai_trace_index", "ai_trace_index_mv", + "audit_log", ]) expect([...v13Names].filter((name) => !currentSchemaNames.has(name))).toEqual([]) const aiTraceIndex = LOCAL_SCHEMA_MANIFEST.objects.find( @@ -310,6 +314,7 @@ describe("local migration registry", () => { "local-0011-to-0012-service-map-edge-quantiles", "local-0012-to-0013-service-operations-discriminators", "local-0013-to-0014-ai-trace-index", + "local-0014-to-0015-audit-log", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -356,7 +361,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 15, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 16, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -1359,6 +1364,7 @@ describe("v10 -> v11 product events module", () => { "local-0011-to-0012-service-map-edge-quantiles", "local-0012-to-0013-service-operations-discriminators", "local-0013-to-0014-ai-trace-index", + "local-0014-to-0015-audit-log", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx index 6c6aab5d0..044c19755 100644 --- a/apps/web/src/components/settings/audit-log-section.tsx +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -178,10 +178,9 @@ export function AuditLogSection() { ))}
- {/* Deliberately not "every change": this records configuration and - access changes plus refused attempts, not reads or telemetry. */}

- Configuration and access changes, from the dashboard, API, and MCP. + Changes, refused attempts, and every read of telemetry or session replays — from the + dashboard, API, and MCP.

@@ -215,7 +214,7 @@ export function AuditLogSection() { No audit log entries - Actions performed by users, API keys, and agents will appear here. + Actions and data reads by users, API keys, and agents will appear here. diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 4417052a8..3525d1ade 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -30,6 +30,7 @@ import { migration_0020_semconv_key_renames } from "./0020_semconv_key_renames" import { migration_0022_service_map_edge_quantiles } from "./0022_service_map_edge_quantiles" import { migration_0023_service_operations_discriminators } from "./0023_service_operations_discriminators" import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" +import { migration_0025_audit_log } from "./0025_audit_log" import { migration_0021_product_events } from "./0021_product_events" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" @@ -46,10 +47,10 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, ]) - expect(migrations.at(-1)).toBe(migration_0024_ai_trace_index) - expect(latestMigrationVersion).toBe(24) + expect(migrations.at(-1)).toBe(migration_0025_audit_log) + expect(latestMigrationVersion).toBe(25) // 0010 and 0014-0020 are read-path only and skipped by the ingest-gating // version; 0021 is not — the gateway writes `session_events`' new identity // columns and `product_events` directly, so a BYO-CH org must apply it @@ -57,7 +58,8 @@ describe("ClickHouse migrations", () => { // tables it touches are MV-populated and the gateway writes neither, and // 0023 is the same: it only adds counter columns to those MV-populated // service-operations rollups. 0024 is read-path only too: `ai_trace_index` - // is MV-populated and the gateway never writes it. + // is MV-populated and the gateway never writes it. 0025 (`audit_log`) is + // written by the API worker through Tinybird, never by the gateway. expect(clickHouseSchemaVersion).toBe("21") expect(migration_0010_search_indexes.requiredForIngest).toBe(false) expect(migration_0014_web_events.requiredForIngest).toBe(false) From 702a2be8c9f1ef2a2efdaebf24214898a26783e4 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 2 Sep 2026 20:50:35 +0200 Subject: [PATCH 09/19] =?UTF-8?q?fix(audit):=20review=20round=20=E2=80=94?= =?UTF-8?q?=20group=20annotations,=20array=20JSONPath,=20dedupe,=20interru?= =?UTF-8?q?pts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `HttpApiGroup.annotate` writes to the group only (endpoint propagation is `annotateEndpoints`), so `withAuditedRead` never saw the action on any group-annotated surface and recorded reads only for the five per-endpoint annotations on the v1 errors group. Both annotation sets are consulted now, endpoint first, and a test drives a group-annotated endpoint, a v2 replay endpoint, an unannotated endpoint, and a failing handler through the wrapper. `ChangedFields` needed the `[:]` JSONPath suffix every other Events-API array column declares; without it the datasource rejects or quarantines rows. The endpoint label used `endpoint.name` (the class name) instead of `identifier`. The comments claimed `LIMIT … BY Id` dedupe the query never had; the service now drops a repeated id within a page and the comments say what actually happens. `recordEventAudit` used `catchCause`, which also swallows interrupts; it catches failures and defects like `record` does. The DateTime64 decoder accepts an ISO rendering, `occurredAtMs` is `Schema.Finite`, and two stale comments about the Postgres table and self-hosted routing are corrected. Regenerated schema artifacts re-pin the local v15 identity. --- apps/api/src/routes/v1/organizations.http.ts | 4 +- .../api/src/services/audit/AuditLogService.ts | 6 +- .../src/services/audit/audit-access.test.ts | 149 ++++++++++++++++++ apps/api/src/services/audit/audit-access.ts | 8 +- apps/api/src/services/audit/audit-event.ts | 9 +- .../errors/ErrorIssueWorkflowService.ts | 7 +- apps/cli/src/server/local-schema-history.ts | 2 +- apps/cli/src/server/schema-identity.ts | 2 +- apps/cli/src/server/schema/local-inserts.json | 2 +- .../src/server/schema/local-schema-v15.sql | 2 +- apps/cli/src/server/schema/local-schema.sql | 2 +- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- .../clickhouse/migrations/0025_audit_log.ts | 6 +- .../domain/src/generated/clickhouse-schema.ts | 2 +- .../generated/tinybird-project-manifest.ts | 4 +- packages/domain/src/tinybird/datasources.ts | 8 +- .../query-engine/src/ch/queries/audit-log.ts | 8 +- 17 files changed, 193 insertions(+), 30 deletions(-) create mode 100644 apps/api/src/services/audit/audit-access.test.ts diff --git a/apps/api/src/routes/v1/organizations.http.ts b/apps/api/src/routes/v1/organizations.http.ts index 6c427c446..7b8dfb4b5 100644 --- a/apps/api/src/routes/v1/organizations.http.ts +++ b/apps/api/src/routes/v1/organizations.http.ts @@ -13,8 +13,8 @@ export const HttpOrganizationsLive = HttpApiBuilder.group(MapleApi, "organizatio const tenant = yield* CurrentTenant.Context const deleted = yield* organizationService.delete(tenant.orgId, tenant.roles) // Recorded after the fact so a refused delete cannot leave an entry - // claiming the org is gone. The row outlives the org: nothing - // cascades `audit_log_entries`, which is the point of a trail. + // claiming the org is gone. The entry outlives the org: the audit + // log is never cascaded, which is the point of a trail. yield* recordHttpAudit("organization.deleted") return deleted }), diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index dd53852fe..7913977f9 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -279,7 +279,7 @@ export class AuditLogService extends Context.Service + const entries = yield* Effect.forEach(rows, (row) => decodeStoredAuditLogEntry(row).pipe( Effect.map((decoded) => storedRowToEntry(orgId, decoded)), Effect.mapError((error) => @@ -290,6 +290,10 @@ export class AuditLogService extends Context.Service() + return entries.filter((entry) => !seen.has(entry.id) && seen.add(entry.id) !== undefined) }) return { record, list } diff --git a/apps/api/src/services/audit/audit-access.test.ts b/apps/api/src/services/audit/audit-access.test.ts new file mode 100644 index 000000000..c0bcc4575 --- /dev/null +++ b/apps/api/src/services/audit/audit-access.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "@effect/vitest" +import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { CurrentTenant, MapleInternalApi } from "@maple/domain/http" +import { MapleApiV2 } from "@maple/domain/http/v2" +import type { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" +import { OrgId, UserId } from "@maple/domain/primitives" +import { Effect, Result, Schema } from "effect" +import { TestClock } from "effect/testing" +import { CurrentAuditActor } from "@/services/auth/audit-actor" +import { makeMemoryAuditLog } from "./AuditLogService" +import { auditAttribution, recordRawSqlAudit, withAuditedRead } from "./audit-access" +import { AuditLogService } from "./AuditLogService" + +const ORG = Schema.decodeUnknownSync(OrgId)("org_audit_access_test") +const USER = Schema.decodeUnknownSync(UserId)("user_audit_access_test") + +/** The `{ group, endpoint }` a security middleware receives for one endpoint. */ +const endpointOf = (api: { readonly groups: Record }, group: string, name: string) => { + const found = api.groups[group] + if (found === undefined) throw new Error(`no group ${group}`) + const endpoint = found.endpoints[name] as HttpApiEndpoint.Top | undefined + if (endpoint === undefined) throw new Error(`no endpoint ${group}.${name}`) + return { group: found, endpoint } +} + +const request = (method: string, url: string, body?: string) => + HttpServerRequest.fromWeb( + new Request(`https://api.test${url}`, { + method, + headers: { "cf-ray": "ray-1", "cf-connecting-ip": "203.0.113.7" }, + ...(body !== undefined ? { body } : undefined), + }), + ) + +class HandlerFailure extends Schema.TaggedError()("HandlerFailure", { + message: Schema.String, +}) {} + +const subject = { + orgId: ORG, + actor: { type: "user" as const, userId: USER }, + source: "dashboard" as const, +} + +describe("withAuditedRead", () => { + it.effect("records a telemetry read for an endpoint whose GROUP carries the annotation", () => + Effect.gen(function* () { + const audit = makeMemoryAuditLog() + const req = request("POST", "/internal/query-engine/execute-batch?x=1", '{"requests":[]}') + const options = endpointOf(MapleInternalApi, "queryEngine", "executeBatch") + // The handler reads the body first, exactly as a real one would. + const handler = req.text.pipe(Effect.map(() => HttpServerResponse.empty({ status: 200 }))) + yield* withAuditedRead(audit, req, options, subject)(handler) + + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries).toHaveLength(1) + const entry = entries[0]! + expect(entry.action).toBe("telemetry.read") + expect(entry.userId).toBe(USER) + expect(entry.requestId).toBe("ray-1") + expect(entry.metadata).toMatchObject({ + endpoint: "queryEngine.executeBatch", + method: "POST", + status: 200, + body: '{"requests":[]}', + }) + }), + ) + + it.effect("records session replay reads on the v2 group and nothing for unannotated endpoints", () => + Effect.gen(function* () { + const audit = makeMemoryAuditLog() + const ok = Effect.succeed(HttpServerResponse.empty({ status: 200 })) + yield* withAuditedRead(audit, request("GET", "/v2/session_replays/s1"), endpointOf(MapleApiV2, "sessionReplays", "retrieve"), subject)(ok) + yield* withAuditedRead(audit, request("GET", "/v2/api_keys"), endpointOf(MapleApiV2, "apiKeys", "list"), subject)(ok) + + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries.map((entry) => entry.action)).toEqual(["session_replay.read"]) + }), + ) + + it.effect("still records an attempted read when the handler fails", () => + Effect.gen(function* () { + const audit = makeMemoryAuditLog() + const failing = Effect.fail(new HandlerFailure({ message: "boom" })) + const outcome = yield* withAuditedRead( + audit, + request("GET", "/v2/traces/t1"), + endpointOf(MapleApiV2, "traces", "retrieve"), + subject, + )(failing).pipe(Effect.result) + expect(Result.isFailure(outcome)).toBe(true) + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries).toHaveLength(1) + expect(entries[0]!.metadata).toMatchObject({ status: 0, endpoint: "traces.retrieve" }) + }), + ) +}) + +describe("auditAttribution", () => { + it("attributes an agent tenant to the agent acting for the user", () => { + const actorId = Schema.decodeUnknownSync(Schema.String)("actor_1") + const attribution = auditAttribution( + { orgId: ORG, userId: USER, actorId: actorId as never, mcpClientName: "claude-code" }, + { type: "api_key", source: "mcp" }, + ) + expect(attribution).toEqual({ + actor: { type: "agent", actorId, userId: USER, label: "claude-code" }, + source: "mcp", + }) + }) + + it("keeps a system token as system regardless of the tenant", () => { + expect(auditAttribution({ orgId: ORG, userId: USER }, { type: "system", source: "system" })).toEqual({ + actor: { type: "system" }, + source: "system", + }) + }) +}) + +describe("recordRawSqlAudit", () => { + it.effect("records a refused statement as denied and an executed one with its row count", () => + Effect.gen(function* () { + const audit = yield* AuditLogService + const base = { + tenant: { orgId: ORG, userId: USER }, + sql: "SELECT 1", + context: "mcp.run_sql", + startTime: "2026-08-29 09:00:00", + endTime: "2026-08-29 10:00:00", + } + yield* recordRawSqlAudit({ ...base, result: { _tag: "rejected", reason: "missing $__orgFilter" } }) + yield* TestClock.adjust("1 second") + yield* recordRawSqlAudit({ ...base, result: { _tag: "rows", rowCount: 3 } }) + + const entries = yield* audit.list(ORG, { limit: 10, offset: 0 }) + expect(entries.map((entry) => [entry.action, entry.outcome])).toEqual([ + ["telemetry.sql_executed", "allowed"], + ["telemetry.sql_executed", "denied"], + ]) + expect(entries[1]!.denialReason).toBe("missing $__orgFilter") + expect(entries[0]!.metadata).toMatchObject({ sql: "SELECT 1", row_count: 3, context: "mcp.run_sql" }) + expect(entries[0]!.source).toBe("mcp") + }).pipe( + Effect.provideService(CurrentAuditActor, { type: "api_key", source: "mcp" }), + Effect.provide(AuditLogService.layerMemory), + ), + ) +}) diff --git a/apps/api/src/services/audit/audit-access.ts b/apps/api/src/services/audit/audit-access.ts index aef9b0f15..2848529ff 100644 --- a/apps/api/src/services/audit/audit-access.ts +++ b/apps/api/src/services/audit/audit-access.ts @@ -98,7 +98,11 @@ export const withAuditedRead = ( httpEffect: Effect.Effect, ): Effect.Effect => { - const action = Context.get(options.endpoint.annotations, AuditedRead) + // A group-level `.annotate` lands on the group only (endpoint propagation + // is `annotateEndpoints`), so both are consulted; the endpoint wins. + const action = + Context.get(options.endpoint.annotations, AuditedRead) ?? + Context.get(options.group.annotations, AuditedRead) if (action === undefined) return httpEffect const record = (status: number) => Effect.gen(function* () { @@ -114,7 +118,7 @@ export const withAuditedRead = source: subject.source, action, metadata: { - endpoint: `${options.group.identifier}.${options.endpoint.name}`, + endpoint: `${options.group.identifier}.${options.endpoint.identifier}`, method: request.method, path: request.url, status, diff --git a/apps/api/src/services/audit/audit-event.ts b/apps/api/src/services/audit/audit-event.ts index 7c935e03e..620fe774c 100644 --- a/apps/api/src/services/audit/audit-event.ts +++ b/apps/api/src/services/audit/audit-event.ts @@ -34,7 +34,7 @@ export class AuditLogEvent extends Schema.Class("AuditLogEvent")( requestId: Schema.optionalKey(Schema.String), originIp: Schema.optionalKey(Schema.String), originCountry: Schema.optionalKey(Schema.String), - occurredAtMs: Schema.Number, + occurredAtMs: Schema.Finite, }) {} export const decodeAuditLogEvent = Schema.decodeUnknownEffect(AuditLogEvent) @@ -142,12 +142,15 @@ const nullableText = emptyAsNull(Schema.String) /** JSON document columns: `''` when absent, otherwise a JSON string of `schema`. */ const jsonDocument = (schema: S) => emptyAsNull(Schema.fromJsonString(schema)) -/** `YYYY-MM-DD HH:mm:ss.SSS` (UTC, as the warehouse emits DateTime64) ⇄ `Date`. */ +/** + * `YYYY-MM-DD HH:mm:ss.SSS` (UTC, as the warehouse emits DateTime64) ⇄ `Date`; + * an ISO rendering with `T`/`Z` is accepted as-is should a backend emit one. + */ const warehouseDateTime = Schema.String.pipe( Schema.decodeTo( Schema.Date, SchemaTransformation.transform({ - decode: (value: string) => new Date(`${value.replace(" ", "T")}Z`), + decode: (value: string) => new Date(/[TZ]/.test(value) ? value : `${value.replace(" ", "T")}Z`), encode: (value: Date) => msToWarehouseDateTime64(value.getTime()), }), ), diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts index e45c2b3c4..dfd280d75 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts @@ -507,8 +507,11 @@ const make: Effect.Effect< }, }) }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Issue event audit write failed", { issueId, cause }), + // Typed failures and defects only — an interrupt must propagate so + // fiber teardown never triggers a stray write. + Effect.catch((error) => Effect.logWarning("Issue event audit write failed", { issueId, cause: error })), + Effect.catchDefect((defect) => + Effect.logWarning("Issue event audit write failed", { issueId, cause: defect }), ), ) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 7a97abeec..95ad7fd9a 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -152,6 +152,6 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje fingerprint: "9dcc61f3d2522826", digest: "9dcc61f3d2522826070ac2063c777d34ec4b1f4492d4ac6663729c21c29a4495", manifestDigest: "f7f9bf825e84924e9d6e0c86712eba5db249db9b8ceb07010d6926ad736bc5e7", - projectRevision: "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748", + projectRevision: "20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf", }), ] as const) diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 6558cb153..df6661b92 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -37,7 +37,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748" + "20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 3a8b95ad7..5508c281b 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748", + "projectRevision": "20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v15.sql b/apps/cli/src/server/schema/local-schema-v15.sql index 26586d01c..55560bad0 100644 --- a/apps/cli/src/server/schema/local-schema-v15.sql +++ b/apps/cli/src/server/schema/local-schema-v15.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748 +-- projectRevision: 20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf -- localSchemaVersion: 15 CREATE TABLE IF NOT EXISTS ai_trace_index ( diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 26586d01c..55560bad0 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748 +-- projectRevision: 20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf -- localSchemaVersion: 15 CREATE TABLE IF NOT EXISTS ai_trace_index ( diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index c62e02e9a..709120f0c 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748"; +pub const PROJECT_REVISION: &str = "20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/clickhouse/migrations/0025_audit_log.ts b/packages/domain/src/clickhouse/migrations/0025_audit_log.ts index 261066cb3..78b0e840e 100644 --- a/packages/domain/src/clickhouse/migrations/0025_audit_log.ts +++ b/packages/domain/src/clickhouse/migrations/0025_audit_log.ts @@ -3,9 +3,9 @@ * * Written only by the API worker through the managed Tinybird pipeline and read * only by the admin-gated `GET /v2/audit_log`. It ships in the migration set so - * self-hosted deployments (where the "managed" pipeline IS this ClickHouse) have - * the table; a BYO-ClickHouse org never reads or writes it — reads are pinned to - * the managed route (`INGEST_PINNED_TABLES`). + * every ClickHouse the schema is applied to mirrors the managed table; a + * BYO-ClickHouse org never reads or writes it — reads are pinned to the managed + * route (`INGEST_PINNED_TABLES`). * * `requiredForIngest: false`: the ingest gateway writes nothing here, so the * table's presence must not gate an org's ingest readiness. diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index ad04b90d6..a96823520 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748" as const +export const projectRevision = "20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_trace_index (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SessionId String,\n VendorId LowCardinality(String),\n ServiceName LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 684248d67..9319df858 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "6530ae04f3c8560e07eabf4644ec16e32cc54c14eacdfebe52c40dbb58943748" as const +export const projectRevision = "20753c5593ff6ab808b536d455b944d5cc300ae0eed841dfa951ff998dcefbaf" as const export const datasources = [ { @@ -27,7 +27,7 @@ export const datasources = [ { name: "audit_log", content: - 'DESCRIPTION >\n Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n RecordedAt DateTime64(3) `json:$.RecordedAt`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n UserId String `json:$.UserId`,\n ApiKeyId String `json:$.ApiKeyId`,\n ActorId String `json:$.ActorId`,\n ActorLabel String `json:$.ActorLabel`,\n AffectedUserId String `json:$.AffectedUserId`,\n Source LowCardinality(String) `json:$.Source`,\n Action LowCardinality(String) `json:$.Action`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n DenialReason String `json:$.DenialReason`,\n ResourceType LowCardinality(String) `json:$.ResourceType`,\n ResourceId String `json:$.ResourceId`,\n ChangedFields Array(String) `json:$.ChangedFields`,\n Changes String `json:$.Changes`,\n Metadata String `json:$.Metadata`,\n RequestId String `json:$.RequestId`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 2190 DAY"', + 'DESCRIPTION >\n Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n RecordedAt DateTime64(3) `json:$.RecordedAt`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n UserId String `json:$.UserId`,\n ApiKeyId String `json:$.ApiKeyId`,\n ActorId String `json:$.ActorId`,\n ActorLabel String `json:$.ActorLabel`,\n AffectedUserId String `json:$.AffectedUserId`,\n Source LowCardinality(String) `json:$.Source`,\n Action LowCardinality(String) `json:$.Action`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n DenialReason String `json:$.DenialReason`,\n ResourceType LowCardinality(String) `json:$.ResourceType`,\n ResourceId String `json:$.ResourceId`,\n ChangedFields Array(String) `json:$.ChangedFields[:]`,\n Changes String `json:$.Changes`,\n Metadata String `json:$.Metadata`,\n RequestId String `json:$.RequestId`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 2190 DAY"', }, { name: "error_events", diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index bc8901784..b7012ce30 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1594,7 +1594,8 @@ export const auditLog = defineDatasource("audit_log", { DenialReason: t.string(), ResourceType: t.string().lowCardinality(), ResourceId: t.string(), - ChangedFields: t.array(t.string()), + // `[:]` is what lets the Events API map a JSON array onto Array(String). + ChangedFields: column(t.array(t.string()), { jsonPath: "$.ChangedFields[:]" }), Changes: t.string(), Metadata: t.string(), RequestId: t.string(), @@ -1602,8 +1603,9 @@ export const auditLog = defineDatasource("audit_log", { OriginCountry: t.string().lowCardinality(), }, // ReplacingMergeTree keyed on the entry id makes queue redelivery idempotent: - // a second delivery of the same event collapses at merge time instead of - // showing as a duplicate row. Reads dedupe the (rare) pre-merge window too. + // a second delivery of the same event collapses at the next merge. Until + // then a page can carry both copies; `AuditLogService.list` drops the + // repeat by id. engine: engine.replacingMergeTree({ partitionKey: "toYYYYMM(OccurredAt)", sortingKey: ["OrgId", "OccurredAt", "Id"], diff --git a/packages/query-engine/src/ch/queries/audit-log.ts b/packages/query-engine/src/ch/queries/audit-log.ts index e08ddc9ef..307c8885b 100644 --- a/packages/query-engine/src/ch/queries/audit-log.ts +++ b/packages/query-engine/src/ch/queries/audit-log.ts @@ -25,12 +25,10 @@ export interface AuditLogEntriesOpts { } /** - * One org's audit log, newest first, offset-paginated. - * - * `LIMIT … BY Id` collapses a redelivered entry that ReplacingMergeTree has not - * merged yet, so a page never shows the same entry twice. Pinned to the managed + * One org's audit log, newest first, offset-paginated. Pinned to the managed * route: the table is written through `ingest` and does not exist in a BYO - * ClickHouse. + * ClickHouse. A redelivered entry ReplacingMergeTree has not merged yet can + * appear twice here; the service collapses it by id. */ export function auditLogEntriesQuery(opts: AuditLogEntriesOpts) { return from(AuditLog) From c0ac24c834e8ae77a877ef848fac13faf120281a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 13:04:36 +0200 Subject: [PATCH 10/19] perf(domain): mark @maple/domain side-effect free so the barrels tree-shake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The web bundle budget went red on this branch at 651.0 KB against a 650.0 KB ceiling — main sits at 648.3 KB, so the audit log's +2.7 KB is what tipped it. None of that 2.7 KB is the settings tab: no audit module is in the static graph. It is the contract. `@maple/domain/http/v2`'s barrel re-exports `audit-log`, and startup modules (error-issues, anomalies, alert form-utils) import that barrel, so the entry schema and the V2 group definition landed in two startup chunks. Without `sideEffects: false` the bundler must assume every module behind a barrel matters and keeps all of it — so any new export taxes startup for everyone, whoever adds it. domain is schemas, contracts and branded types: no import-time global mutation, no prototype patching, nothing registered on load. Declaring that lets rolldown drop what a given entry does not reach. 651.0 -> 648.9 KB gzip. domain 693 tests, web 2391 pass. Co-Authored-By: Claude Opus 5 --- packages/domain/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/domain/package.json b/packages/domain/package.json index a464a3756..ce89d63d7 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -3,6 +3,7 @@ "version": "0.0.0", "private": true, "type": "module", + "sideEffects": false, "exports": { ".": "./src/index.ts", "./anticipated-errors": "./src/anticipated-errors.ts", From aa79ccd6f7280ec774335641264d0219ee2e46b1 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 13:22:36 +0200 Subject: [PATCH 11/19] feat(audit): name the actor instead of printing its id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every user row rendered as `user_3BfcmIS3bUNV6BfA…` and every API-key row as `key_jCZgpwAYQ3…`, because nothing on the HTTP path ever set `actorLabel` — only the MCP client name and the system surface did. The two actor kinds want opposite treatments, so they get them: - An API key already has its name on the row auth just resolved, so the name is frozen onto the entry at write time, denials included. That is the property an audit trail wants: a key that is revoked an hour later still reads as "Grafana exporter" on the request that was refused. Free — no extra lookup. - A dashboard session has no name to freeze. Clerk's claims carry none, and resolving one per write would put a directory call on every telemetry read — the hottest audited path there is. Those rows are labelled when the log is read, one directory call per page, on an admin-only screen. Reading the log never fails because the directory does: an unconfigured (self-hosted, no Clerk) or unavailable directory logs a warning and the entries keep their ids. A member who has since left the org is unnameable by construction and renders as an id — deliberately, since that is often exactly the actor being looked for. `listMembers` is promoted onto OrgMembersService for this: `resolveMembers` fails the whole call when any id is not a current member, which is the wrong shape for labelling historical records. Verified against the local stack: a refused key now reads "Grafana exporter". apps/api 215 files / 2623 tests, typecheck 40/40, lint clean. Co-Authored-By: Claude Opus 5 --- apps/api/src/routes/v2/audit-log.http.test.ts | 33 +++++++++ apps/api/src/routes/v2/audit-log.http.ts | 68 +++++++++++++++++-- apps/api/src/routes/v2/v2-test-support.ts | 18 ++++- .../src/services/audit/audit-access.test.ts | 20 ++++++ apps/api/src/services/audit/audit-access.ts | 1 + .../services/auth/ApiAuthorizationLayer.ts | 9 ++- .../services/auth/ApiAuthorizationV2Layer.ts | 9 ++- apps/api/src/services/auth/audit-actor.ts | 7 ++ apps/api/src/services/auth/audit-denial.ts | 3 + apps/api/src/services/org/ApiKeysService.ts | 3 + .../api/src/services/org/OrgMembersService.ts | 13 +++- 11 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 apps/api/src/routes/v2/audit-log.http.test.ts diff --git a/apps/api/src/routes/v2/audit-log.http.test.ts b/apps/api/src/routes/v2/audit-log.http.test.ts new file mode 100644 index 000000000..ee7a945d1 --- /dev/null +++ b/apps/api/src/routes/v2/audit-log.http.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "@effect/vitest" +import { Schema } from "effect" +import { UserId } from "@maple/domain/primitives" +import type { AuditLogEntry } from "@/services/audit/audit-event" +import { actorDisplayName } from "./audit-log.http" + +const USER = Schema.decodeUnknownSync(UserId)("user_audit_route_test") + +type Named = Pick + +describe("actorDisplayName", () => { + it("prefers the label frozen at write time over the current directory", () => { + const row: Named = { actorLabel: "Deploy bot", userId: USER } + expect(actorDisplayName(row, new Map([[USER, "Ada Lovelace"]]))).toBe("Deploy bot") + }) + + it("names a dashboard actor from the directory", () => { + const row: Named = { actorLabel: null, userId: USER } + expect(actorDisplayName(row, new Map([[USER, "Ada Lovelace"]]))).toBe("Ada Lovelace") + }) + + // A member who has since left the org is exactly the actor an audit reader + // cares about, so an unresolvable id must still render as itself rather than + // dropping the row or erroring. + it("leaves a departed member unnamed", () => { + const row: Named = { actorLabel: null, userId: USER } + expect(actorDisplayName(row, new Map())).toBeNull() + }) + + it("has nothing to name for a system entry", () => { + expect(actorDisplayName({ actorLabel: null, userId: null }, new Map())).toBeNull() + }) +}) diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts index 450b89e04..e1735d75f 100644 --- a/apps/api/src/routes/v2/audit-log.http.ts +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -1,6 +1,6 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { CurrentTenant } from "@maple/domain/http" -import { ActorId, ApiKeyId, UserId } from "@maple/domain/primitives" +import { ActorId, ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" import { decodePublicId, encodePublicId, @@ -15,11 +15,18 @@ import type { V2AuditLogEntry } from "@maple/domain/http/v2" import type { AuditLogEntry } from "@/services/audit/audit-event" import { Effect, Option, Schema } from "effect" import { AuditLogService } from "@/services/audit/AuditLogService" +import { OrgMembersService } from "@/services/org/OrgMembersService" import { requireAdmin } from "@/services/auth/auth" import type { AuditLogListFilters } from "@/services/audit/AuditLogService" const adminOnly = () => V2InsufficientPermissions.make("Only org admins can read the audit log") +/** No directory, no names — the entries still carry every id they were written with. */ +const unnamed = (cause: unknown) => + Effect.logWarning("Audit log: member directory unavailable; entries keep their ids", { + cause, + }).pipe(Effect.as(new Map())) + const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) const decodeUserIdOption = Schema.decodeUnknownOption(UserId) @@ -69,7 +76,29 @@ const publicActorId = (row: AuditLogEntry): string | null => { } } -const toV2AuditLogEntry = (row: AuditLogEntry): V2AuditLogEntry => ({ +/** + * The name to show for one entry: the label frozen at write time when there is + * one, else the current directory name for the acting user, else nothing — + * which renders as the id the entry already carries. + */ +export const actorDisplayName = ( + row: Pick, + names: ReadonlyMap, +): string | null => row.actorLabel ?? (row.userId === null ? null : (names.get(row.userId) ?? null)) + +/** + * Name the humans. An API key freezes its name into `actorLabel` when the entry + * is written, which is what an audit trail wants — the name the credential had + * at the time. A dashboard session has nothing to freeze: Clerk's claims carry + * no name, and resolving one per write would put a directory call on every + * telemetry read. So user rows are labelled here, from the directory as it + * stands, and an id that no longer belongs to a member simply keeps showing as + * an id. + */ +const toV2AuditLogEntry = ( + row: AuditLogEntry, + names: ReadonlyMap, +): V2AuditLogEntry => ({ id: row.id, object: "audit_log_entry", action: row.action, @@ -77,7 +106,7 @@ const toV2AuditLogEntry = (row: AuditLogEntry): V2AuditLogEntry => ({ denial_reason: row.denialReason, actor_type: row.actorType, actor_id: publicActorId(row), - actor_name: row.actorLabel, + actor_name: actorDisplayName(row, names), affected_user: row.affectedUserId, source: row.source, resource_type: row.resourceType, @@ -94,6 +123,29 @@ const toV2AuditLogEntry = (row: AuditLogEntry): V2AuditLogEntry => ({ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", (handlers) => Effect.gen(function* () { const audit = yield* AuditLogService + const members = yield* OrgMembersService + + /** + * Display names for the user ids on one page, or none at all: a directory + * that is unconfigured (self-hosted without Clerk) or briefly unavailable + * must never turn reading the audit log into an error. Ids still render. + * + * `catch` + `catchDefect` rather than `catchCause`, which would also + * swallow the interrupt that tears this request down. + */ + const displayNames = (orgId: OrgId) => + members.listMembers(orgId).pipe( + Effect.map( + (all) => + new Map( + all.flatMap((member) => + member.name === null ? [] : [[member.userId, member.name] as const], + ), + ), + ), + Effect.catch((error) => unnamed(error)), + Effect.catchDefect((defect) => unnamed(defect)), + ) return handlers.handle("list", ({ query }) => Effect.gen(function* () { @@ -135,7 +187,15 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( limit, offset, }) - .pipe(Effect.map((rows) => rows.map(toV2AuditLogEntry))), + .pipe( + Effect.flatMap((rows) => + rows.length === 0 + ? Effect.succeed([]) + : displayNames(tenant.orgId).pipe( + Effect.map((names) => rows.map((row) => toV2AuditLogEntry(row, names))), + ), + ), + ), ) return { object: "list" as const, ...page } }), diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index 2b7d4252d..a077fbfa0 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -43,6 +43,7 @@ import { HttpV2OrganizationLive } from "./organization.http" import { HttpV2InstrumentationRecommendationsLive } from "./recommendations.http" import { HttpV2AuditLogLive } from "./audit-log.http" import { AuditLogService } from "@/services/audit/AuditLogService" +import { OrgMembersService } from "@/services/org/OrgMembersService" import { HttpV2ScrapeTargetsLive } from "./scrape-targets.http" import { HttpV2SessionReplaysLive } from "./session-replays.http" import { HttpV2InstrumentationAuditLive } from "./setup-audit.http" @@ -66,6 +67,15 @@ import { HttpV2WidgetCredentialsLive } from "./widget-credentials.http" * the groups it does not exercise. */ +/** + * An empty workspace directory: the audit log falls back to rendering ids, + * which is the same shape a self-hosted deployment without Clerk sees. + */ +export const OrgMembersServiceStubLayer = Layer.succeed(OrgMembersService, { + listMembers: () => Effect.succeed([]), + resolveMembers: () => Effect.succeed([]), +}) + export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2ApiKeysLive, HttpV2SlackIntegrationsLive, @@ -78,8 +88,12 @@ export const AllV2GroupLayersLive = Layer.mergeAll( HttpV2IngestKeysLive, HttpV2ErrorIssuesLive, HttpV2AttributeMappingsLive, - // Real service, no stub: it needs only the Database every harness already provides. - HttpV2AuditLogLive.pipe(Layer.provide(AuditLogService.layerMemory)), + // Real service, no stub: it needs only the Database every harness already + // provides. The member directory IS stubbed — the route asks it for display + // names, and every harness would otherwise reach for Clerk. + HttpV2AuditLogLive.pipe( + Layer.provide(Layer.merge(AuditLogService.layerMemory, OrgMembersServiceStubLayer)), + ), HttpV2ScrapeTargetsLive, HttpV2InstrumentationRecommendationsLive, HttpV2InstrumentationAuditLive, diff --git a/apps/api/src/services/audit/audit-access.test.ts b/apps/api/src/services/audit/audit-access.test.ts index c0bcc4575..9aec60ee1 100644 --- a/apps/api/src/services/audit/audit-access.test.ts +++ b/apps/api/src/services/audit/audit-access.test.ts @@ -110,6 +110,26 @@ describe("auditAttribution", () => { }) }) + it("freezes the API key's name onto the entry", () => { + const apiKeyId = Schema.decodeUnknownSync(Schema.String)("key_1") + expect( + auditAttribution( + { orgId: ORG, userId: USER }, + { type: "api_key", apiKeyId: apiKeyId as never, label: "Deploy bot", source: "api" }, + ), + ).toEqual({ + actor: { type: "api_key", userId: USER, apiKeyId, label: "Deploy bot" }, + source: "api", + }) + }) + + it("leaves a dashboard session unlabelled — its name is resolved when the log is read", () => { + expect(auditAttribution({ orgId: ORG, userId: USER }, { type: "user", source: "dashboard" })).toEqual({ + actor: { type: "user", userId: USER }, + source: "dashboard", + }) + }) + it("keeps a system token as system regardless of the tenant", () => { expect(auditAttribution({ orgId: ORG, userId: USER }, { type: "system", source: "system" })).toEqual({ actor: { type: "system" }, diff --git a/apps/api/src/services/audit/audit-access.ts b/apps/api/src/services/audit/audit-access.ts index 2848529ff..0eac00e86 100644 --- a/apps/api/src/services/audit/audit-access.ts +++ b/apps/api/src/services/audit/audit-access.ts @@ -72,6 +72,7 @@ export const auditAttribution = (tenant: AuditTenant, info: AuditActorInfo | und type: info?.type ?? "user", userId: tenant.userId, ...(info?.apiKeyId !== undefined ? { apiKeyId: info.apiKeyId } : undefined), + ...(info?.label !== undefined ? { label: info.label } : undefined), }, source: info?.source ?? "dashboard", } diff --git a/apps/api/src/services/auth/ApiAuthorizationLayer.ts b/apps/api/src/services/auth/ApiAuthorizationLayer.ts index 01ca47857..6079b6e27 100644 --- a/apps/api/src/services/auth/ApiAuthorizationLayer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationLayer.ts @@ -56,6 +56,7 @@ export const ApiAuthorizationLayer = Layer.effect( orgId: resolved.orgId, userId: resolved.userId, apiKeyId: resolved.keyId, + apiKeyName: resolved.name, denialReason, }) if (resolved.kind !== "standard") { @@ -86,11 +87,17 @@ export const ApiAuthorizationLayer = Layer.effect( Effect.provideService(CurrentAuditActor, { type: "api_key", apiKeyId: resolved.keyId, + label: resolved.name, source: "api", }), withAuditedRead(audit, request, options, { orgId: resolved.orgId, - actor: { type: "api_key", userId: resolved.userId, apiKeyId: resolved.keyId }, + actor: { + type: "api_key", + userId: resolved.userId, + apiKeyId: resolved.keyId, + label: resolved.name, + }, source: "api", }), ) diff --git a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts index 67d8a71ad..4192671c4 100644 --- a/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts +++ b/apps/api/src/services/auth/ApiAuthorizationV2Layer.ts @@ -106,6 +106,7 @@ export const ApiAuthorizationV2Layer = Layer.effect( orgId: resolved.orgId, userId: resolved.userId, apiKeyId: resolved.keyId, + apiKeyName: resolved.name, denialReason, }) // Deny-list, not an allow-list: `mcp` keys are minted through a @@ -203,12 +204,18 @@ export const ApiAuthorizationV2Layer = Layer.effect( Effect.provideService(CurrentAuditActor, { type: "api_key", apiKeyId: resolved.keyId, + label: resolved.name, source: "api", }), // Telemetry and replay reads are recorded (see `AuditedRead`). withAuditedRead(audit, request, options, { orgId: resolved.orgId, - actor: { type: "api_key", userId: resolved.userId, apiKeyId: resolved.keyId }, + actor: { + type: "api_key", + userId: resolved.userId, + apiKeyId: resolved.keyId, + label: resolved.name, + }, source: "api", }), ) diff --git a/apps/api/src/services/auth/audit-actor.ts b/apps/api/src/services/auth/audit-actor.ts index 43147417e..bb917cb45 100644 --- a/apps/api/src/services/auth/audit-actor.ts +++ b/apps/api/src/services/auth/audit-actor.ts @@ -12,6 +12,13 @@ export interface AuditActorInfo { /** `system` is Maple itself acting through an internal service token. */ readonly type: "user" | "api_key" | "system" readonly apiKeyId?: ApiKeyId + /** + * A display name for the credential, frozen into the entry. Set for API + * keys, whose name is already on the row auth resolved; a dashboard session + * has no name to carry (Clerk's claims hold none), so those entries are + * labelled when the log is read. + */ + readonly label?: string /** The surface the request arrived through, recorded as the entry's `source`. */ readonly source: AuditLogSource } diff --git a/apps/api/src/services/auth/audit-denial.ts b/apps/api/src/services/auth/audit-denial.ts index f8dd9352e..44b1d0c6f 100644 --- a/apps/api/src/services/auth/audit-denial.ts +++ b/apps/api/src/services/auth/audit-denial.ts @@ -44,6 +44,8 @@ export interface ApiDenialInput { readonly orgId: OrgId readonly userId: UserId readonly apiKeyId: ApiKeyId + /** The key's name, frozen onto the entry — a refused key is often revoked next. */ + readonly apiKeyName: string readonly denialReason: string } @@ -75,6 +77,7 @@ export const recordApiDenial = ( type: "api_key", userId: input.userId, apiKeyId: input.apiKeyId, + label: input.apiKeyName, }, source: "api", action: "api.request", diff --git a/apps/api/src/services/org/ApiKeysService.ts b/apps/api/src/services/org/ApiKeysService.ts index 673ca3097..0898a6285 100644 --- a/apps/api/src/services/org/ApiKeysService.ts +++ b/apps/api/src/services/org/ApiKeysService.ts @@ -26,6 +26,8 @@ export interface ResolvedApiKey { readonly orgId: OrgId readonly userId: UserId readonly keyId: ApiKeyId + /** The key's display name, frozen into audit entries at write time. */ + readonly name: string readonly kind: ApiKeyKind readonly metadataJson: string | null /** v2 scope strings; null = legacy full access. */ @@ -594,6 +596,7 @@ export class ApiKeysService extends Context.Service()("@maple/ap orgId: row.value.orgId, userId: row.value.createdBy, keyId: row.value.id, + name: row.value.name, kind: row.value.kind, metadataJson: row.value.metadataJson == null ? null : JSON.stringify(row.value.metadataJson), scopes: row.value.scopes ?? null, diff --git a/apps/api/src/services/org/OrgMembersService.ts b/apps/api/src/services/org/OrgMembersService.ts index 45e10c1b0..6a69c5ca3 100644 --- a/apps/api/src/services/org/OrgMembersService.ts +++ b/apps/api/src/services/org/OrgMembersService.ts @@ -22,6 +22,17 @@ export interface OrgMembersServiceApi { * Fails when any id is not a member of the org, or when member resolution * is unavailable (self-hosted mode without Clerk). */ + /** + * Every member of the org. Unlike {@link resolveMembers} this answers for + * the directory as it is now, so a caller labelling historical records gets + * the members it can name and nothing for ids that have since left. + */ + readonly listMembers: ( + orgId: OrgId, + ) => Effect.Effect< + ReadonlyArray, + AlertMemberDirectoryNotConfiguredError | AlertMemberDirectoryUnavailableError + > readonly resolveMembers: ( orgId: OrgId, userIds: ReadonlyArray, @@ -119,7 +130,7 @@ const make = Effect.gen(function* () { return resolved }) - return { resolveMembers } satisfies OrgMembersServiceApi + return { listMembers, resolveMembers } satisfies OrgMembersServiceApi }) export class OrgMembersService extends Context.Service()( From 27f8d70efe8e50c3c4e70cbdd47607e0b7ca6665 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 13:47:25 +0200 Subject: [PATCH 12/19] perf(audit): the queue is the write path; stop buying durability with response time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed `queue.send` fell back to writing the row straight to the warehouse, inline, before the response went out. That put a second network round trip on the response path at exactly the wrong moment: a Queues brown-out already costs the 2s send timeout, and the fallback then added a Tinybird write on top — every audited read, which is every dashboard telemetry query, turned slow while the platform was degraded. The queue already is the durability story: retries, then the DLQ. A send that cannot be made now logs and drops the entry rather than charging the caller for it. `writeDirect` stays for runtimes with no queue binding at all — local dev, crons, the consumer itself — none of which are serving a response. The test that pinned the old behaviour now pins the new one: a failed send writes nothing to the warehouse and still does not fail the caller. apps/api 215 files / 2623 tests, typecheck clean, lint clean. Co-Authored-By: Claude Opus 5 --- .../services/audit/AuditLogService.test.ts | 9 ++-- .../api/src/services/audit/AuditLogService.ts | 43 ++++++++++++------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/apps/api/src/services/audit/AuditLogService.test.ts b/apps/api/src/services/audit/AuditLogService.test.ts index 67297f6ed..808f64e6f 100644 --- a/apps/api/src/services/audit/AuditLogService.test.ts +++ b/apps/api/src/services/audit/AuditLogService.test.ts @@ -140,7 +140,7 @@ describe("AuditLogService (warehouse-backed)", () => { }), ) - it.effect("degrades to a direct write when the queue send fails", () => + it.effect("drops the entry rather than writing to the warehouse on the response path", () => Effect.gen(function* () { const warehouse = recordingWarehouse() yield* Effect.gen(function* () { @@ -167,8 +167,11 @@ describe("AuditLogService (warehouse-backed)", () => { ), ), ) - expect(warehouse.ingested).toHaveLength(1) - expect(warehouse.ingested[0]!.rows[0]!.Action).toBe("alert_rule.updated") + // The queue owns durability (retries, DLQ). A failed send must not buy a + // second network round trip with the caller's response time, which is + // exactly when the platform is already degraded — the caller still + // succeeds, and the loss is in the logs. + expect(warehouse.ingested).toHaveLength(0) }), ) diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index 5f4e51efa..866a3c77d 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -41,8 +41,9 @@ export const AUDIT_LOG_DATASOURCE = "audit_log" /** * `queue.send` sits on the response path of every mutation, denial, and - * audited read. A healthy send is tens of ms; 2s bounds a stalling broker - * before the entry degrades to a direct warehouse write. + * audited read, and is the only thing that does. A healthy send is tens of ms; + * 2s bounds a stalling broker, after which the entry is dropped with a warning + * rather than charged to the response as a second network round trip. */ export const AUDIT_QUEUE_SEND_TIMEOUT = "2 seconds" @@ -98,9 +99,11 @@ export interface AuditLogListFilters { export interface AuditLogServiceApi { /** - * Append one entry, durably: published to the audit events queue when the - * binding is present (the consumer performs the warehouse write, retried by - * the queue), written straight to the warehouse otherwise (local dev, crons). + * Append one entry: published to the audit events queue when the binding is + * present (the consumer performs the warehouse write, retried by the queue + * and parked in the DLQ after that), written straight to the warehouse only + * where there is no queue at all — local dev, crons, the consumer itself — + * none of which are serving a response. * Never fails: an action that succeeded must not 500 because its audit * write did not — terminal failures are logged and swallowed. */ @@ -233,11 +236,20 @@ export class AuditLogService extends Context.Service - Effect.logWarning("Audit queue send failed; writing directly", { cause: error }).pipe( - Effect.andThen(writeDirect(event)), - ) - + /** + * One queue send, and that is the whole write path wherever a queue + * exists. It used to fall back to a direct warehouse write when the + * send failed — which put a second network round trip on the response + * path exactly when the platform was already degraded, turning a + * Queues brown-out into slow requests for every audited read. The + * queue is the durability story (retries, DLQ); if the send itself + * cannot be made, the entry is lost and says so in the logs rather + * than being bought at the caller's expense. + * + * `writeDirect` remains for runtimes with no queue binding at all — + * local dev, crons, and the consumer itself — none of which are + * serving a response. + */ const publish = (event: AuditLogEvent) => queue === undefined ? writeDirect(event) @@ -246,15 +258,14 @@ export class AuditLogService extends Context.Service new AuditQueueSendError({ message: "Audit queue send failed", cause }), }).pipe( - // A Queues brown-out that stalls (rather than rejects) must not + // A Queues brown-out that stalls rather than rejects must not // hang the response: 2s is far above a healthy send's latency - // yet bounds the worst case before the direct-write fallback. + // yet bounds the worst case. Effect.timeout(AUDIT_QUEUE_SEND_TIMEOUT), Effect.catchTag("TimeoutError", (error) => - Effect.fail(new AuditQueueSendError({ message: "Audit queue send timed out", cause: error })), - ), - Effect.catchTag("@maple/api/services/audit/AuditQueueSendError", (error) => - fallbackToDirect(event, error), + Effect.fail( + new AuditQueueSendError({ message: "Audit queue send timed out", cause: error }), + ), ), ) From 455990adfe78e887b2aaa406fac243a64ba51323 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 16:06:27 +0200 Subject: [PATCH 13/19] =?UTF-8?q?feat(audit):=20show=20the=20person=20?= =?UTF-8?q?=E2=80=94=20name=20and=20avatar=20for=20user=20actors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actor column named users but still showed them as strangers: a display name only when Clerk had first/last set, no face, and an opaque `user_…` otherwise. - `OrgMember` carries `imageUrl`, which the membership list already returns — no extra call — and the v2 entry gains `actor_avatar_url`, resolved when the log is read like the name is. - A member with no name set falls back to their email rather than an id. That is what the rest of the product shows and what an admin actually recognises; `user_3Bfcm…` is the last resort, not the second one. - The row renders a 16px avatar with initials behind it, for people only. Naming is now gated on the actor being a person, which fixes a real misattribution: every API-key and agent entry also carries a `userId` — whoever minted the credential — so the directory lookup was printing that person's name next to an "API key" badge on actions they did not take. Caught by looking at the page: a denial from a key read "API key · David Ambrus". Keys keep the name frozen at write time; older entries without one show their id, which is honest. apps/api 215 files / 2623 tests, domain 693, typecheck 40/40, lint clean. Web bundle 649.0 KB against the 650.0 KB budget. Co-Authored-By: Claude Opus 5 --- apps/api/src/routes/v2/audit-log.http.test.ts | 49 ++++++++++--- apps/api/src/routes/v2/audit-log.http.ts | 68 +++++++++++++++---- .../api/src/services/org/OrgMembersService.ts | 4 +- .../components/settings/audit-log-section.tsx | 24 +++++++ packages/domain/src/http/v2/audit-log.ts | 6 ++ 5 files changed, 128 insertions(+), 23 deletions(-) diff --git a/apps/api/src/routes/v2/audit-log.http.test.ts b/apps/api/src/routes/v2/audit-log.http.test.ts index ee7a945d1..fffde2bf2 100644 --- a/apps/api/src/routes/v2/audit-log.http.test.ts +++ b/apps/api/src/routes/v2/audit-log.http.test.ts @@ -2,32 +2,65 @@ import { describe, expect, it } from "@effect/vitest" import { Schema } from "effect" import { UserId } from "@maple/domain/primitives" import type { AuditLogEntry } from "@/services/audit/audit-event" -import { actorDisplayName } from "./audit-log.http" +import { actorAvatarUrl, actorDisplayName, type ActorProfile } from "./audit-log.http" const USER = Schema.decodeUnknownSync(UserId)("user_audit_route_test") -type Named = Pick +const directory = (profile: ActorProfile) => new Map([[USER, profile]]) +const ada: ActorProfile = { name: "Ada Lovelace", imageUrl: "https://img.test/ada.png" } + +type Row = Pick describe("actorDisplayName", () => { it("prefers the label frozen at write time over the current directory", () => { - const row: Named = { actorLabel: "Deploy bot", userId: USER } - expect(actorDisplayName(row, new Map([[USER, "Ada Lovelace"]]))).toBe("Deploy bot") + const row: Row = { actorLabel: "Deploy bot", userId: USER, actorType: "api_key" } + expect(actorDisplayName(row, directory(ada))).toBe("Deploy bot") }) it("names a dashboard actor from the directory", () => { - const row: Named = { actorLabel: null, userId: USER } - expect(actorDisplayName(row, new Map([[USER, "Ada Lovelace"]]))).toBe("Ada Lovelace") + const row: Row = { actorLabel: null, userId: USER, actorType: "user" } + expect(actorDisplayName(row, directory(ada))).toBe("Ada Lovelace") }) // A member who has since left the org is exactly the actor an audit reader // cares about, so an unresolvable id must still render as itself rather than // dropping the row or erroring. it("leaves a departed member unnamed", () => { - const row: Named = { actorLabel: null, userId: USER } + const row: Row = { actorLabel: null, userId: USER, actorType: "user" } expect(actorDisplayName(row, new Map())).toBeNull() }) + // An API-key row carries the minting user's id. Naming it from the directory + // would print a person's name on an action a key took. + it("does not lend a minting user's name to their API key", () => { + const row: Row = { actorLabel: null, userId: USER, actorType: "api_key" } + expect(actorDisplayName(row, directory(ada))).toBeNull() + }) + it("has nothing to name for a system entry", () => { - expect(actorDisplayName({ actorLabel: null, userId: null }, new Map())).toBeNull() + expect(actorDisplayName({ actorLabel: null, userId: null, actorType: "system" }, new Map())).toBeNull() + }) +}) + +describe("actorAvatarUrl", () => { + it("shows the directory avatar for a user", () => { + expect(actorAvatarUrl({ actorType: "user", userId: USER }, directory(ada))).toBe( + "https://img.test/ada.png", + ) + }) + + // The key acted, not the person who minted it: showing that person's face + // would misattribute the action to a human who may not have been involved. + it("gives an API key no face even though the entry carries a user id", () => { + expect(actorAvatarUrl({ actorType: "api_key", userId: USER }, directory(ada))).toBeNull() + }) + + it("has none for a member the directory does not know", () => { + expect(actorAvatarUrl({ actorType: "user", userId: USER }, new Map())).toBeNull() + }) + + it("tolerates a member with no avatar", () => { + const noFace: ActorProfile = { name: "Ada Lovelace", imageUrl: null } + expect(actorAvatarUrl({ actorType: "user", userId: USER }, directory(noFace))).toBeNull() }) }) diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts index e1735d75f..428fac493 100644 --- a/apps/api/src/routes/v2/audit-log.http.ts +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -25,7 +25,7 @@ const adminOnly = () => V2InsufficientPermissions.make("Only org admins can read const unnamed = (cause: unknown) => Effect.logWarning("Audit log: member directory unavailable; entries keep their ids", { cause, - }).pipe(Effect.as(new Map())) + }).pipe(Effect.as(new Map())) const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) @@ -76,15 +76,43 @@ const publicActorId = (row: AuditLogEntry): string | null => { } } +/** What the workspace directory can tell us about one member. */ +export interface ActorProfile { + readonly name: string + readonly imageUrl: string | null +} + /** * The name to show for one entry: the label frozen at write time when there is - * one, else the current directory name for the acting user, else nothing — - * which renders as the id the entry already carries. + * one, else the current directory name — but only for a user actor, and only + * ever their own. + * + * Every API-key and agent row also carries a `userId`, the person who minted + * the credential. Naming those rows from the directory puts that person's name + * on an action a key took, which is precisely the attribution an audit log + * exists to keep straight. Entries written before keys carried their name show + * an id, which is honest. */ export const actorDisplayName = ( - row: Pick, - names: ReadonlyMap, -): string | null => row.actorLabel ?? (row.userId === null ? null : (names.get(row.userId) ?? null)) + row: Pick, + directory: ReadonlyMap, +): string | null => + row.actorLabel ?? + (row.actorType !== "user" || row.userId === null + ? null + : (directory.get(row.userId)?.name ?? null)) + +/** + * The avatar, for user actors only. An API key or an agent has no face, and a + * departed member has no directory entry to take one from. + */ +export const actorAvatarUrl = ( + row: Pick, + directory: ReadonlyMap, +): string | null => + row.actorType !== "user" || row.userId === null + ? null + : (directory.get(row.userId)?.imageUrl ?? null) /** * Name the humans. An API key freezes its name into `actorLabel` when the entry @@ -97,7 +125,7 @@ export const actorDisplayName = ( */ const toV2AuditLogEntry = ( row: AuditLogEntry, - names: ReadonlyMap, + directory: ReadonlyMap, ): V2AuditLogEntry => ({ id: row.id, object: "audit_log_entry", @@ -106,7 +134,8 @@ const toV2AuditLogEntry = ( denial_reason: row.denialReason, actor_type: row.actorType, actor_id: publicActorId(row), - actor_name: actorDisplayName(row, names), + actor_name: actorDisplayName(row, directory), + actor_avatar_url: actorAvatarUrl(row, directory), affected_user: row.affectedUserId, source: row.source, resource_type: row.resourceType, @@ -126,20 +155,29 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( const members = yield* OrgMembersService /** - * Display names for the user ids on one page, or none at all: a directory + * Names and avatars for the acting users, or nothing at all: a directory * that is unconfigured (self-hosted without Clerk) or briefly unavailable * must never turn reading the audit log into an error. Ids still render. * + * A member with no name set falls back to their email, which is what the + * rest of the product shows and what an admin reading an audit trail + * actually recognises — an opaque `user_…` is the last resort, not the + * second one. + * * `catch` + `catchDefect` rather than `catchCause`, which would also * swallow the interrupt that tears this request down. */ - const displayNames = (orgId: OrgId) => + const directory = (orgId: OrgId) => members.listMembers(orgId).pipe( Effect.map( (all) => new Map( - all.flatMap((member) => - member.name === null ? [] : [[member.userId, member.name] as const], + all.map( + (member) => + [ + member.userId, + { name: member.name ?? member.email, imageUrl: member.imageUrl }, + ] as const, ), ), ), @@ -191,8 +229,10 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( Effect.flatMap((rows) => rows.length === 0 ? Effect.succeed([]) - : displayNames(tenant.orgId).pipe( - Effect.map((names) => rows.map((row) => toV2AuditLogEntry(row, names))), + : directory(tenant.orgId).pipe( + Effect.map((known) => + rows.map((row) => toV2AuditLogEntry(row, known)), + ), ), ), ), diff --git a/apps/api/src/services/org/OrgMembersService.ts b/apps/api/src/services/org/OrgMembersService.ts index 6a69c5ca3..cb88b2492 100644 --- a/apps/api/src/services/org/OrgMembersService.ts +++ b/apps/api/src/services/org/OrgMembersService.ts @@ -14,6 +14,8 @@ export interface OrgMember { readonly userId: string readonly email: string readonly name: string | null + /** Provider-hosted avatar; Clerk serves one for every user, initials included. */ + readonly imageUrl: string | null } export interface OrgMembersServiceApi { @@ -91,7 +93,7 @@ const make = Effect.gen(function* () { [member.publicUserData?.firstName, member.publicUserData?.lastName] .filter(Boolean) .join(" ") || null - all.push({ userId, email, name }) + all.push({ userId, email, name, imageUrl: member.publicUserData?.imageUrl ?? null }) } offset += page.data.length if (offset >= page.totalCount || page.data.length === 0) break diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx index 044c19755..2b6e50f67 100644 --- a/apps/web/src/components/settings/audit-log-section.tsx +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -5,6 +5,7 @@ import { useState, type ReactNode } from "react" import { Result, useAtomRefresh, useAtomValue } from "@/lib/effect-atom" import { auditLogPageAtom } from "@/lib/services/atoms/audit-log-atoms" +import { Avatar, AvatarFallback, AvatarImage } from "@maple/ui/components/ui/avatar" import { Badge } from "@maple/ui/components/ui/badge" import { Button } from "@maple/ui/components/ui/button" import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" @@ -48,6 +49,20 @@ const COL = { } const COL_HEADER = "text-muted-foreground/70 font-mono text-[10px] uppercase tracking-[0.12em]" +/** + * Up to two initials from a display name, for the moment before the avatar + * loads and for the members Clerk serves no picture for. An email falls back to + * its first letter rather than parsing a local part that is rarely a name. + */ +function initialsOf(label: string): string { + const words = label.trim().split(/\s+/).filter(Boolean) + if (words.length === 0 || label.includes("@")) return label.slice(0, 1).toUpperCase() + return words + .slice(0, 2) + .map((word) => word.slice(0, 1).toUpperCase()) + .join("") +} + function formatDateTime(value: string): string { return new Date(value).toLocaleString(undefined, { month: "short", @@ -302,6 +317,15 @@ function AuditLogRow({ entry }: { entry: V2AuditLogEntry }) { {badge.label} + {/* A face only for people. A key or an agent carries a user id too — + the person who minted it — and showing them here would credit a + human for something they may not have done. */} + {entry.actor_type === "user" && entry.actor_name !== null && ( + + {entry.actor_avatar_url !== null && } + {initialsOf(entry.actor_name)} + + )} {actorLabel} diff --git a/packages/domain/src/http/v2/audit-log.ts b/packages/domain/src/http/v2/audit-log.ts index d4091f716..059532c95 100644 --- a/packages/domain/src/http/v2/audit-log.ts +++ b/packages/domain/src/http/v2/audit-log.ts @@ -61,6 +61,7 @@ const auditLogEntryExample = { actor_type: "user", actor_id: "user_2fj3K9dLqWm8xYbT", actor_name: "David", + actor_avatar_url: "https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCJ9", affected_user: null, source: "dashboard", resource_type: "alert_rule", @@ -101,6 +102,11 @@ export const V2AuditLogEntry = Schema.Struct({ "Display name of the actor at the time of the action (agent name, API key name, …), or `null` when none was recorded.", examples: ["David"], }), + actor_avatar_url: Schema.NullOr(Schema.String).annotate({ + description: + "Avatar for a user actor, resolved from the workspace directory when the log is read. `null` for every non-user actor, and for a user who is no longer a member.", + examples: ["https://img.clerk.com/eyJ0eXBlIjoi…"], + }), affected_user: Schema.NullOr(Schema.String).annotate({ description: "The `user_…` ID of the user the action was performed on (e.g. a removed member), when different from the actor; otherwise `null`.", From 47c6a6170c66d0e3194e64b6b0a82c6c29cf5343 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 16:52:25 +0200 Subject: [PATCH 14/19] feat(web): a person is a face, not the word "User" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actor column read `[User] David Ambrus` on every human row — a badge whose only job was to say "this is a person", next to a face and a name that already said it. The avatar now replaces the badge for user actors; keys, agents and system entries keep theirs, because they have no face to identify them by. Two details the change turns up: - The avatar is Clerk's own `publicUserData.imageUrl`, which Clerk serves for every member — a real `img.clerk.com` URL rendering their initials when they uploaded no picture, so a row is never a blank circle. Verified in the page: the img loads (naturalWidth 128), it is not the local fallback. - A row we could not name falls back to the raw `user_…` id, and taking initials off that produced a confident "U". It shows "?" instead — an unknown member, not someone whose name begins with U. Web bundle 649.0 KB against the 650.0 KB budget. Co-Authored-By: Claude Opus 5 --- .../components/settings/audit-log-section.tsx | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx index 2b6e50f67..43be44a53 100644 --- a/apps/web/src/components/settings/audit-log-section.tsx +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -55,6 +55,9 @@ const COL_HEADER = "text-muted-foreground/70 font-mono text-[10px] uppercase tra * its first letter rather than parsing a local part that is rarely a name. */ function initialsOf(label: string): string { + // A row we could not name falls back to the raw `user_…` id; "U" would read + // as a name it is not. + if (label.startsWith("user_")) return "?" const words = label.trim().split(/\s+/).filter(Boolean) if (words.length === 0 || label.includes("@")) return label.slice(0, 1).toUpperCase() return words @@ -314,17 +317,21 @@ function AuditLogRow({ entry }: { entry: V2AuditLogEntry }) { {formatRelativeTime(entry.occurred_at)}
- - {badge.label} - - {/* A face only for people. A key or an agent carries a user id too — - the person who minted it — and showing them here would credit a - human for something they may not have done. */} - {entry.actor_type === "user" && entry.actor_name !== null && ( + {/* A person is shown as a face, not as the word "User" — the avatar + already says which kind of actor this is, and the name says who. + Keys, agents and system entries have no face and keep their badge. + The face is never lent to a key or an agent: those rows carry the + minting user's id too, and wearing it would credit a human for + something they may not have done. */} + {entry.actor_type === "user" ? ( {entry.actor_avatar_url !== null && } - {initialsOf(entry.actor_name)} + {initialsOf(actorLabel)} + ) : ( + + {badge.label} + )} {actorLabel} From df63d1492b96e865b998b0a7254c20708c79f759 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 17:28:23 +0200 Subject: [PATCH 15/19] refactor(audit): use the primitives and helpers that already existed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sweep of what this branch added, against what the repo already has. **A third copy of a shared formatter.** `msToWarehouseDateTime64` in apps/api hand-rolled `YYYY-MM-DD HH:mm:ss.SSS` with manual UTC padding — byte-identical to `formatWarehouseDateTimeMs`, which query-engine has exported all along for exactly this ("a minority of callers deliberately keep the fractional part (DateTime64 columns…)"). Deleted, and the three call sites now go through the shared one. **No brand on the millisecond shape.** `WarehouseDateTime` is branded so it "cannot be produced by string manipulation"; its `DateTime64(3)` sibling had nothing, so a hand-built whole-second or ISO `T`/`Z` string type-checked into an ingest row — the first collapses the sub-second ordering a DateTime64 sort key exists to keep, the second is rejected by the Events API's JSONPath parser and dropped by the warehouse rather than by anything in front of it. Added `WarehouseDateTime64` + `warehouseDateTime64(epochMs)` beside them, mirroring the existing pair. Worth noting why the builder's `dateTime64` codec is not the answer here: its encoder is `.slice(0, 19)`, which truncates the milliseconds. The audit query sidesteps it already by binding `param.dateTimeString`, so the precision does survive into the SQL — but nothing said so. **`Effect.catchCause` in the queue consumer.** The one this branch had not yet converted. v4's catchCause catches interruption too, so a batch interrupted by a deploy would be counted as a failed attempt and pushed toward the DLQ for something that never failed. Now `catch` + `catchDefect`, leaving the interrupt to unwind — the batch stays unacked and the platform redelivers it. A defect now retries as well, which is new behaviour and has a test that fails when the `catchDefect` is removed. **Duplicated request forensics.** `audit-denial` rebuilt the cf-ray/cf-connecting-ip/cf-ipcountry block that `httpRequestForensics` already provides three lines away. Left alone deliberately: `try/finally` at the worker queue boundary (outside Effect, and identical to its sibling consumers); `Date.parse` on `since`/`until` in the v2 route, whose input is already constrained by the `Timestamp` schema so it cannot be NaN. apps/api 215 files / 2629 tests, query-engine 62, typecheck 40/40, lint clean. Co-Authored-By: Claude Opus 5 --- apps/api/src/audit-events-runtime.test.ts | 20 +++++++++++++ apps/api/src/audit-events-runtime.ts | 15 ++++++++-- apps/api/src/platform/time.ts | 10 ------- apps/api/src/services/alerts/AlertsService.ts | 6 ++-- .../api/src/services/audit/AuditLogService.ts | 6 ++-- apps/api/src/services/audit/audit-event.ts | 17 ++++++----- apps/api/src/services/auth/audit-denial.ts | 12 ++------ packages/query-engine/src/datetime.ts | 28 +++++++++++++++++++ 8 files changed, 79 insertions(+), 35 deletions(-) diff --git a/apps/api/src/audit-events-runtime.test.ts b/apps/api/src/audit-events-runtime.test.ts index 958617264..84496b2b0 100644 --- a/apps/api/src/audit-events-runtime.test.ts +++ b/apps/api/src/audit-events-runtime.test.ts @@ -42,6 +42,14 @@ const message = (body: unknown, attempts: number) => { const batchOf = (...messages: ReadonlyArray<{ readonly message: unknown }>) => ({ messages: messages.map((entry) => entry.message) }) as never +/** A warehouse whose `ingest` dies rather than failing — an unexpected defect. */ +const dyingWarehouse = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: () => Effect.die(new Error("ingest exploded")), + }), +) + /** A warehouse whose `ingest` records each call, or fails every call. */ const warehouse = (fail = false) => { const written: Array<{ orgId: string; rows: ReadonlyArray }> = [] @@ -105,6 +113,18 @@ describe("processAuditEventsBatch", () => { }), ) + // A typed failure retries; so must a defect. Catching only the failure + // channel would let an unexpected throw escape the consumer, and Cloudflare + // treats a consumer that neither acked nor retried as a retry anyway — but + // silently, with no log and no DLQ accounting. + it.effect("retries when the write dies instead of failing", () => + Effect.gen(function* () { + const defect = message(event("77777777-7777-4777-8777-777777777777"), 2) + yield* processAuditEventsBatch(batchOf(defect)).pipe(Effect.provide(dyingWarehouse)) + expect(defect.calls).toEqual(["retry"]) + }), + ) + // A message that cannot decode will never decode. Retrying only burns the // attempts that would otherwise carry a recoverable message to the DLQ. it.effect("acks a malformed message instead of retrying it forever", () => diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts index 223e6c786..18ebe247d 100644 --- a/apps/api/src/audit-events-runtime.ts +++ b/apps/api/src/audit-events-runtime.ts @@ -157,8 +157,19 @@ export const processAuditEventsBatch = (batch: MessageBatch) => }), ), Effect.withSpan("auditEvents.writeOrgBatch", { attributes: { orgId, rows: group.length } }), - Effect.catchCause((cause) => - Effect.forEach(group, ({ message }) => retryOrExhaust(message, cause), { + // `catch` + `catchDefect`, never `catchCause`: v4's catchCause also + // catches interruption, and an interrupted batch (a deploy, an + // isolate torn down) would then be counted as a failed attempt — + // pushing messages toward the DLQ for something that never failed. + // Left uncaught, the interrupt simply leaves the batch unacked and + // the platform redelivers it. + Effect.catch((error) => + Effect.forEach(group, ({ message }) => retryOrExhaust(message, error), { + discard: true, + }), + ), + Effect.catchDefect((defect) => + Effect.forEach(group, ({ message }) => retryOrExhaust(message, defect), { discard: true, }), ), diff --git a/apps/api/src/platform/time.ts b/apps/api/src/platform/time.ts index 6b22ad5c8..e4b74ebca 100644 --- a/apps/api/src/platform/time.ts +++ b/apps/api/src/platform/time.ts @@ -33,13 +33,3 @@ export function dateToMs(date: Date | null | undefined): number | null { return date === null || date === undefined ? null : date.getTime() } -/** - * Tinybird `DateTime64(3)` wire format for `ingest` rows: `YYYY-MM-DD HH:mm:ss.SSS`, - * UTC, no zone. Every direct warehouse write (alert checks, audit entries) sends - * timestamps this way; ISO's `T`/`Z` is rejected by the Events API JSONPath parser. - */ -export function msToWarehouseDateTime64(ms: number): string { - const d = new Date(ms) - const pad = (n: number, w = 2) => n.toString().padStart(w, "0") - return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` -} diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index b17f37125..22eb24d5d 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,4 +1,4 @@ -import { formatWarehouseDateTime, snapAlertWindowEndMs } from "@maple/query-engine" +import { formatWarehouseDateTime, snapAlertWindowEndMs, warehouseDateTime64 } from "@maple/query-engine" import { AlertComparator as AlertComparatorSchema, AlertDeliveryError, @@ -89,7 +89,7 @@ import { EmailService } from "@/platform/EmailService" import { Env } from "@/platform/Env" import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" import { makeDbExecute } from "@/platform/db-execute" -import { dateToMs, msToDate, msToSqlTimestamp, msToWarehouseDateTime64 } from "@/platform/time" +import { dateToMs, msToDate, msToSqlTimestamp } from "@/platform/time" import { makePersistenceError } from "./alert-persistence" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import type { GroupedAlertObservation } from "@maple/query-engine/runtime" @@ -283,7 +283,7 @@ export const interleaveAlertRulesByOrg = ( return fair } -const toIngestDateTime64 = msToWarehouseDateTime64 +const toIngestDateTime64 = warehouseDateTime64 const compareThreshold = ( value: number, diff --git a/apps/api/src/services/audit/AuditLogService.ts b/apps/api/src/services/audit/AuditLogService.ts index 866a3c77d..f563d96f3 100644 --- a/apps/api/src/services/audit/AuditLogService.ts +++ b/apps/api/src/services/audit/AuditLogService.ts @@ -8,7 +8,7 @@ import * as CH from "@maple/query-engine/ch" import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import type { Queue } from "@cloudflare/workers-types" import { WorkerEnvironment } from "@maple/infra/worker-runtime" -import { msToWarehouseDateTime64 } from "@/platform/time" +import { warehouseDateTime64 } from "@maple/query-engine/datetime" import { systemTenant } from "@/services/alerts/system-tenant" import { CurrentAuditActor } from "@/services/auth/audit-actor" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" @@ -167,8 +167,8 @@ const neverFail = (action: string) => (write: Effect.Effect) => /** Which optional filters bind, and the parameter values behind them. */ const listQueryInputs = (orgId: OrgId, filters: AuditLogListFilters) => { - const since = filters.sinceMs === undefined ? undefined : msToWarehouseDateTime64(filters.sinceMs) - const until = filters.untilMs === undefined ? undefined : msToWarehouseDateTime64(filters.untilMs) + const since = filters.sinceMs === undefined ? undefined : warehouseDateTime64(filters.sinceMs) + const until = filters.untilMs === undefined ? undefined : warehouseDateTime64(filters.untilMs) const opts: CH.AuditLogEntriesOpts = { actorType: filters.actorType !== undefined, userId: filters.userId !== undefined, diff --git a/apps/api/src/services/audit/audit-event.ts b/apps/api/src/services/audit/audit-event.ts index 620fe774c..f5f0f2a4b 100644 --- a/apps/api/src/services/audit/audit-event.ts +++ b/apps/api/src/services/audit/audit-event.ts @@ -7,7 +7,8 @@ import { import { ActorId, ApiKeyId, AuditLogEntryId, OrgId, UserId } from "@maple/domain/primitives" import type { AuditLogRow } from "@maple/domain/tinybird" import { Schema, SchemaTransformation } from "effect" -import { msToDate, msToWarehouseDateTime64 } from "@/platform/time" +import { warehouseDateTime64 } from "@maple/query-engine/datetime" +import { msToDate } from "@/platform/time" /** * The serialized audit event as it travels the audit queue. `occurredAtMs` is @@ -74,8 +75,8 @@ export interface AuditLogEntry { export const auditEventToRow = (event: AuditLogEvent, recordedAtMs: number): AuditLogRow => ({ OrgId: event.orgId, Id: event.id, - OccurredAt: msToWarehouseDateTime64(event.occurredAtMs), - RecordedAt: msToWarehouseDateTime64(recordedAtMs), + OccurredAt: warehouseDateTime64(event.occurredAtMs), + RecordedAt: warehouseDateTime64(recordedAtMs), ActorType: event.actorType, UserId: event.userId ?? "", ApiKeyId: event.apiKeyId ?? "", @@ -146,12 +147,14 @@ const jsonDocument = (schema: S) => emptyAsNull(Schema.fro * `YYYY-MM-DD HH:mm:ss.SSS` (UTC, as the warehouse emits DateTime64) ⇄ `Date`; * an ISO rendering with `T`/`Z` is accepted as-is should a backend emit one. */ -const warehouseDateTime = Schema.String.pipe( +const warehouseDateTime64Column = Schema.String.pipe( Schema.decodeTo( Schema.Date, SchemaTransformation.transform({ decode: (value: string) => new Date(/[TZ]/.test(value) ? value : `${value.replace(" ", "T")}Z`), - encode: (value: Date) => msToWarehouseDateTime64(value.getTime()), + // The brand is the minting side's guarantee; a codec encodes to the + // wire type, which is a plain string. + encode: (value: Date): string => warehouseDateTime64(value.getTime()), }), ), ) @@ -162,8 +165,8 @@ const warehouseDateTime = Schema.String.pipe( */ export const StoredAuditLogEntry = Schema.Struct({ id: AuditLogEntryId, - occurredAt: warehouseDateTime, - recordedAt: warehouseDateTime, + occurredAt: warehouseDateTime64Column, + recordedAt: warehouseDateTime64Column, actorType: AuditActorType, userId: emptyAsNull(UserId), apiKeyId: emptyAsNull(ApiKeyId), diff --git a/apps/api/src/services/auth/audit-denial.ts b/apps/api/src/services/auth/audit-denial.ts index 44b1d0c6f..428755efe 100644 --- a/apps/api/src/services/auth/audit-denial.ts +++ b/apps/api/src/services/auth/audit-denial.ts @@ -1,7 +1,7 @@ import { Clock, Effect } from "effect" import type { HttpServerRequest } from "effect/unstable/http" import type { ApiKeyId, OrgId, UserId } from "@maple/domain/primitives" -import type { AuditLogServiceApi } from "@/services/audit/AuditLogService" +import { httpRequestForensics, type AuditLogServiceApi } from "@/services/audit/AuditLogService" /** Suppress duplicate denial rows for the same key/reason within this window. */ export const AUDIT_DENIAL_COALESCE_WINDOW_MS = 60_000 @@ -84,14 +84,6 @@ export const recordApiDenial = ( outcome: "denied", denialReason: input.denialReason, metadata: { method: request.method, path }, - ...(request.headers["cf-ray"] !== undefined - ? { requestId: request.headers["cf-ray"] } - : undefined), - ...(request.headers["cf-connecting-ip"] !== undefined - ? { originIp: request.headers["cf-connecting-ip"] } - : undefined), - ...(request.headers["cf-ipcountry"] !== undefined - ? { originCountry: request.headers["cf-ipcountry"] } - : undefined), + ...httpRequestForensics(request), }) }) diff --git a/packages/query-engine/src/datetime.ts b/packages/query-engine/src/datetime.ts index 25c4eef82..9d0f56195 100644 --- a/packages/query-engine/src/datetime.ts +++ b/packages/query-engine/src/datetime.ts @@ -158,6 +158,34 @@ export const WarehouseDateTime = Schema.String.pipe( }) export type WarehouseDateTime = Schema.Schema.Type +/** + * A warehouse `DateTime64(3)` literal: `YYYY-MM-DD HH:mm:ss.SSS`, UTC, no zone. + * + * The millisecond sibling of {@link WarehouseDateTime}, and branded for the same + * reason: the shapes a hand-built string reaches for — whole seconds, or ISO + * with `T`/`Z` — are both wrong here. Seconds collapse the sub-second ordering + * that a `DateTime64(3)` sort key exists to keep, and the Events API's JSONPath + * parser rejects `T`/`Z` outright, so an ingest row carrying one is dropped by + * the warehouse rather than by any check in front of it. + */ +export const WarehouseDateTime64 = Schema.String.pipe( + Schema.check(Schema.isPattern(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/)), + Schema.brand("@maple/WarehouseDateTime64"), +).annotate({ + title: "WarehouseDateTime64", + description: + "UTC warehouse DateTime64(3) literal, `YYYY-MM-DD HH:mm:ss.SSS` (e.g. `2026-08-25 08:47:52.041`).", +}) +export type WarehouseDateTime64 = Schema.Schema.Type + +/** + * Format epoch milliseconds as a {@link WarehouseDateTime64} — the one + * sanctioned way to mint the brand, mirroring {@link warehouseDateTime}. + */ +export function warehouseDateTime64(epochMs: number): WarehouseDateTime64 { + return WarehouseDateTime64.make(formatWarehouseDateTimeMs(epochMs)) +} + /** * Decodes any accepted timestamp input — ISO-8601 with `Z` or an offset, the * warehouse shape with or without fractional seconds, a bare date — into a From b0aa3e38ba9b65b5b27be0ac1cbcfab53ccee541 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 17:42:52 +0200 Subject: [PATCH 16/19] refactor(audit): fold destination changes into the shared auditDiff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `buildDestinationChanges` was the last update handler still assembling its diff by hand — 30 lines of loop plus two lookup helpers, doing what `auditDiff` was built to do for every other resource. The bespoke part it needed and the shared helper lacked: a field the response does not echo but which is no secret — a Slack channel id, a Telegram chat id, the Hazel org handles. `writeOnly` withholds a credential's value as ``; these are simply not knowable from the document, and read better as ``. So `auditDiff` grows an `opaque` bucket alongside `writeOnly`, and destinations become a declaration: three diffed fields, five credentials, eight handles. That kills a quiet hazard. The old loop derived audit field names by regex from the *internal* camelCase request (`channelId` -> `channel_id`), so a wire key rename would silently change what the log recorded, and the secret set was keyed on internal names too — a rename there would have silently un-redacted a credential. The spec is keyed on the wire names the payload actually carries. Equivalence checked rather than assumed: old and new agree on all nine representative payloads (every destination type, an unchanged field, a rename alongside a rotation, and a missing pre-update document), modulo field order, which moves from request-key order to a stable declaration order. The old code had no tests. This adds nine — three for `opaque`, six for the destination spec, including that a webhook URL is withheld rather than diffed. apps/api 215 files / 2637 tests, typecheck 40/40, lint clean. Co-Authored-By: Claude Opus 5 --- .../src/routes/v2/alert-destinations.http.ts | 90 +++++++------------ apps/api/src/routes/v2/audit-changes.test.ts | 79 ++++++++++++++++ apps/api/src/routes/v2/audit-changes.ts | 27 ++++-- 3 files changed, 132 insertions(+), 64 deletions(-) diff --git a/apps/api/src/routes/v2/alert-destinations.http.ts b/apps/api/src/routes/v2/alert-destinations.http.ts index 9f8325bba..a33585d6d 100644 --- a/apps/api/src/routes/v2/alert-destinations.http.ts +++ b/apps/api/src/routes/v2/alert-destinations.http.ts @@ -1,5 +1,6 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" -import type { AlertDestinationDocument, AlertDestinationUpdateRequest, AuditChanges } from "@maple/domain/http" +import type { AlertDestinationDocument, AlertDestinationUpdateRequest } from "@maple/domain/http" +import { auditDiff } from "./audit-changes" import { CurrentTenant, DiscordAlertDestinationConfig, @@ -193,62 +194,35 @@ const toUpdateRequest = (params: V2AlertDestinationUpdateParams): AlertDestinati } /** Credential-bearing config keys; their values must never reach the audit row. */ -const destinationSecretKeys = new Set(["integrationKey", "signingSecret", "url", "webhookUrl", "botToken"]) - -/** Fields of an update that are readable back off the destination document. */ -const destinationObservableValue = ( - doc: AlertDestinationDocument, - key: string, -): string | boolean | ReadonlyArray | null | undefined => { - switch (key) { - case "name": - return doc.name - case "enabled": - return doc.enabled - case "memberUserIds": - return doc.memberUserIds - default: - return undefined - } -} - /** - * Diff an update against the pre/post documents. Secrets are recorded as - * ``; config knobs the wire doc doesn't echo (channel ids, chat ids) - * are recorded as touched with `` placeholders. + * The three update fields a destination document echoes back. Everything else + * an update can carry is either a credential or a provider-side handle the + * document never returns, so the diff can only record that it was touched. */ -const buildDestinationChanges = ( - request: AlertDestinationUpdateRequest, - before: AlertDestinationDocument | undefined, - after: AlertDestinationDocument, -): AuditChanges | undefined => { - const fields: string[] = [] - const beforeOut: Record = {} - const afterOut: Record = {} - for (const key of Object.keys(request)) { - if (key === "type") continue - const wireName = key.replace(/[A-Z]/g, (char) => `_${char.toLowerCase()}`) - if (destinationSecretKeys.has(key)) { - fields.push(wireName) - beforeOut[wireName] = "" - afterOut[wireName] = "" - continue - } - const prev = before === undefined ? undefined : destinationObservableValue(before, key) - const next = destinationObservableValue(after, key) - if (prev === undefined && next === undefined) { - fields.push(wireName) - beforeOut[wireName] = "" - afterOut[wireName] = "" - continue - } - if (JSON.stringify(prev) === JSON.stringify(next)) continue - fields.push(wireName) - beforeOut[wireName] = prev - afterOut[wireName] = next - } - return fields.length === 0 ? undefined : { fields, before: beforeOut, after: afterOut } -} +const destinationAuditView = (doc: AlertDestinationDocument | undefined) => ({ + name: doc?.name, + enabled: doc?.enabled, + member_user_ids: doc?.memberUserIds, +}) + +export const destinationAuditDiff = auditDiff({ + fields: ["name", "enabled", "member_user_ids"], + // A webhook URL is a credential: Discord's carries the token in the path, and + // a plain webhook's can carry one in userinfo or query. + writeOnly: ["integration_key", "signing_secret", "url", "webhook_url", "bot_token"], + // Provider-side handles. Not secret, but not readable back off the document + // either — recorded as touched so the change is not invisible. + opaque: [ + "channel_id", + "channel_name", + "chat_id", + "hazel_organization_id", + "hazel_organization_name", + "hazel_organization_logo_url", + "hazel_channel_id", + "hazel_channel_name", + ], +}) export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "alertDestinations", (handlers) => Effect.gen(function* () { @@ -322,7 +296,11 @@ export const HttpV2AlertDestinationsLive = HttpApiBuilder.group(MapleApiV2, "ale request, ) - const changes = buildDestinationChanges(request, current, updated) + const changes = destinationAuditDiff( + payload, + destinationAuditView(current), + destinationAuditView(updated), + ) yield* recordHttpAudit("alert_destination.updated", { resourceId: updated.id, changes, diff --git a/apps/api/src/routes/v2/audit-changes.test.ts b/apps/api/src/routes/v2/audit-changes.test.ts index a7f31fd58..95fe688d6 100644 --- a/apps/api/src/routes/v2/audit-changes.test.ts +++ b/apps/api/src/routes/v2/audit-changes.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest" import { auditDiff, redactAuditUrl } from "./audit-changes" +import { destinationAuditDiff } from "./alert-destinations.http" const targetDiff = auditDiff({ fields: ["name", "url", "enabled", "labels_json"], @@ -81,3 +82,81 @@ describe("auditDiff", () => { expect(changes?.after).toEqual({ name: "renamed", auth_credentials: "" }) }) }) + +describe("auditDiff opaque fields", () => { + const knobDiff = auditDiff({ + fields: ["name"], + opaque: ["channel_id"], + writeOnly: ["bot_token"], + }) + + it("records a provider-side handle as touched, not as its value", () => { + const changes = knobDiff({ channel_id: "C0123" }, { name: "n" }, { name: "n" }) + expect(changes).toEqual({ + fields: ["channel_id"], + before: { channel_id: "" }, + after: { channel_id: "" }, + }) + // Not secret, but not ours to echo either — the document never returns it. + expect(JSON.stringify(changes)).not.toContain("C0123") + }) + + it("keeps a knob distinct from a credential in the same request", () => { + const changes = knobDiff( + { name: "renamed", channel_id: "C0123", bot_token: "xoxb-secret" }, + { name: "before" }, + { name: "renamed" }, + ) + expect(changes?.fields).toEqual(["name", "bot_token", "channel_id"]) + expect(changes?.after).toEqual({ + name: "renamed", + bot_token: "", + channel_id: "", + }) + }) + + it("says nothing when the request carried neither", () => { + expect(knobDiff({ name: "same" }, { name: "same" }, { name: "same" })).toBeUndefined() + }) +}) + +describe("destinationAuditDiff", () => { + const view = (over: Record = {}) => ({ + name: "Ops", + enabled: true, + member_user_ids: undefined, + ...over, + }) + + it("diffs the fields a destination document echoes", () => { + const changes = destinationAuditDiff({ enabled: false }, view(), view({ enabled: false })) + expect(changes).toEqual({ fields: ["enabled"], before: { enabled: true }, after: { enabled: false } }) + }) + + it("never records a rotated credential's value", () => { + const changes = destinationAuditDiff({ bot_token: "xoxb-secret" }, view(), view()) + expect(changes?.fields).toEqual(["bot_token"]) + expect(JSON.stringify(changes)).not.toContain("xoxb-secret") + }) + + // A webhook URL is the credential for Discord and can carry one for a plain + // webhook, so it is withheld rather than diffed. + it("treats a webhook URL as a credential", () => { + const changes = destinationAuditDiff({ webhook_url: "https://discord.test/api/webhooks/1/tok" }, view(), view()) + expect(changes?.after).toEqual({ webhook_url: "" }) + }) + + it("records a channel move as touched", () => { + const changes = destinationAuditDiff({ channel_id: "C9", channel_name: "#alerts" }, view(), view()) + expect(changes?.fields).toEqual(["channel_id", "channel_name"]) + expect(changes?.after).toEqual({ channel_id: "", channel_name: "" }) + }) + + // The pre-update document is looked up from a list; if it were missing, the + // diff must still record what the request changed rather than nothing. + it("still records a change when the previous document is unknown", () => { + const unknown = { name: undefined, enabled: undefined, member_user_ids: undefined } + const changes = destinationAuditDiff({ name: "Ops" }, unknown, view()) + expect(changes).toEqual({ fields: ["name"], before: { name: undefined }, after: { name: "Ops" } }) + }) +}) diff --git a/apps/api/src/routes/v2/audit-changes.ts b/apps/api/src/routes/v2/audit-changes.ts index 4986d06dc..7f781f7ad 100644 --- a/apps/api/src/routes/v2/audit-changes.ts +++ b/apps/api/src/routes/v2/audit-changes.ts @@ -96,19 +96,25 @@ export const redactAuditUrl = (raw: string): string => { * The spec is declared once next to the resource's wire shape and applied per * request: `fields` are diffed through the wire view, `summarize` replaces a * config blob's value with a static placeholder, `redact` rewrites a value - * (scrape URLs carry tokens), and `writeOnly` records credentials the response - * never echoes as having rotated. `summarize` and `redact` are keyed by + * (scrape URLs carry tokens), `writeOnly` records credentials the response + * never echoes as having rotated, and `opaque` records a knob the response does + * not echo either but which is no secret — a channel id, a chat id — as simply + * touched. `summarize` and `redact` are keyed by * `fields`, so a renamed wire key is a type error rather than a silently * disabled redaction. * * Returns undefined when nothing observable changed, so the caller passes the * result straight through as `changes`. */ +const redactedField = (field: string): readonly [string, string] => [field, ""] +const updatedField = (field: string): readonly [string, string] => [field, ""] + export const auditDiff = (spec: { readonly fields: ReadonlyArray readonly summarize?: Partial> readonly redact?: Partial string>> readonly writeOnly?: ReadonlyArray + readonly opaque?: ReadonlyArray }) => { const redactors: Record string) | undefined> = spec.redact ?? {} @@ -136,14 +142,19 @@ export const auditDiff = (spec: { ) const compacted = diffed === undefined ? undefined : compactAuditChanges(diffed, spec.summarize ?? {}) const observable = compacted === undefined ? undefined : redactChanges(compacted) - // Write-only fields never appear in a response, so their rotation can only - // be inferred from the request carrying them. + // Neither kind appears in a response, so that the request carried them is + // the only evidence they changed. They differ in what may be said about + // them: a credential's value is withheld, a channel id's is simply not + // known here. const present: Record = payload - const rotated = (spec.writeOnly ?? []).filter((field) => present[field] !== undefined) - if (rotated.length === 0) return observable - const placeholders = Object.fromEntries(rotated.map((field) => [field, ""])) + const touched = [ + ...(spec.writeOnly ?? []).filter((field) => present[field] !== undefined).map(redactedField), + ...(spec.opaque ?? []).filter((field) => present[field] !== undefined).map(updatedField), + ] + if (touched.length === 0) return observable + const placeholders = Object.fromEntries(touched) return { - fields: [...(observable?.fields ?? []), ...rotated], + fields: [...(observable?.fields ?? []), ...touched.map(([field]) => field)], before: { ...observable?.before, ...placeholders }, after: { ...observable?.after, ...placeholders }, } From 28bf03f171f359fb668408b6c12f383e53694f21 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 17:53:04 +0200 Subject: [PATCH 17/19] refactor(audit): the house idiom for recovering without swallowing teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Effect.catch` + `Effect.catchDefect` as a pair was mine, and it is not what this repo does. `Effect.catchCause` guarded by `Cause.hasInterruptsOnly` is — ten call sites already, in turn-runner, agent-pass, DigestService, AnomalyDetectionService, EscalationService and PlanetScaleService. One combinator instead of two, and the interrupt exemption is stated in the code rather than in a comment explaining why catchCause was avoided. Both sites this branch introduced now follow it, and both re-raise rather than recover: a request being torn down has no audit page left to label, and an interrupted queue batch must not spend a retry — unacked is enough, the platform redelivers. Causes are logged through `summarizeCause`, which the repo uses in seventy places, instead of dumping the cause object into the log annotation. New test: an interrupted batch retries nothing and exits as a failure. The suite is checked against a mutant — forcing the guard to `true` fails the failure, defect and DLQ cases and leaves only the interrupt case passing. apps/api 215 files / 2638 tests, typecheck 40/40, lint clean (the effect-lint `unnecessary-pipe-chain` rule caught a chained pipe here first). Co-Authored-By: Claude Opus 5 --- apps/api/src/audit-events-runtime.test.ts | 21 ++++++++++++++++ apps/api/src/audit-events-runtime.ts | 29 ++++++++++------------- apps/api/src/routes/v2/audit-log.http.ts | 21 +++++++++------- 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/apps/api/src/audit-events-runtime.test.ts b/apps/api/src/audit-events-runtime.test.ts index 84496b2b0..8ce050216 100644 --- a/apps/api/src/audit-events-runtime.test.ts +++ b/apps/api/src/audit-events-runtime.test.ts @@ -42,6 +42,12 @@ const message = (body: unknown, attempts: number) => { const batchOf = (...messages: ReadonlyArray<{ readonly message: unknown }>) => ({ messages: messages.map((entry) => entry.message) }) as never +/** A warehouse whose `ingest` is interrupted — a deploy tearing the isolate down. */ +const interruptedWarehouse = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ ingest: () => Effect.interrupt }), +) + /** A warehouse whose `ingest` dies rather than failing — an unexpected defect. */ const dyingWarehouse = Layer.succeed( WarehouseQueryService, @@ -125,6 +131,21 @@ describe("processAuditEventsBatch", () => { }), ) + // Interruption is not a failed attempt. Counting it as one would spend the + // message's retry budget — and eventually route it to the DLQ — for a deploy. + // Unacked is enough: the platform redelivers. + it.effect("does not count an interrupted batch as an attempt", () => + Effect.gen(function* () { + const torn = message(event("88888888-8888-4888-8888-888888888888"), 2) + const exit = yield* processAuditEventsBatch(batchOf(torn)).pipe( + Effect.provide(interruptedWarehouse), + Effect.exit, + ) + expect(exit._tag).toBe("Failure") + expect(torn.calls).toEqual([]) + }), + ) + // A message that cannot decode will never decode. Retrying only burns the // attempts that would otherwise carry a recoverable message to the DLQ. it.effect("acks a malformed message instead of retrying it forever", () => diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts index 18ebe247d..1efee2ff9 100644 --- a/apps/api/src/audit-events-runtime.ts +++ b/apps/api/src/audit-events-runtime.ts @@ -4,8 +4,9 @@ import { EdgeCacheService } from "@maple/cache" import { ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/domain/anticipated-errors" import type { OrgId } from "@maple/domain/primitives" import { WorkerConfigProviderLayer, workerEnvironmentLayer } from "@maple/infra/worker-runtime" -import { Clock, Effect, Layer } from "effect" +import { Cause, Clock, Effect, Layer } from "effect" import { CacheBackendLive } from "@/platform/CacheBackendLive" +import { summarizeCause } from "@/platform/describe-cause" import { layerPg } from "@/platform/DatabasePgLive" import { Env } from "@/platform/Env" import { systemTenant } from "@/services/alerts/system-tenant" @@ -157,21 +158,17 @@ export const processAuditEventsBatch = (batch: MessageBatch) => }), ), Effect.withSpan("auditEvents.writeOrgBatch", { attributes: { orgId, rows: group.length } }), - // `catch` + `catchDefect`, never `catchCause`: v4's catchCause also - // catches interruption, and an interrupted batch (a deploy, an - // isolate torn down) would then be counted as a failed attempt — - // pushing messages toward the DLQ for something that never failed. - // Left uncaught, the interrupt simply leaves the batch unacked and - // the platform redelivers it. - Effect.catch((error) => - Effect.forEach(group, ({ message }) => retryOrExhaust(message, error), { - discard: true, - }), - ), - Effect.catchDefect((defect) => - Effect.forEach(group, ({ message }) => retryOrExhaust(message, defect), { - discard: true, - }), + // A failure or a defect is a failed attempt and retries. Interruption + // is not: an interrupted batch (a deploy, an isolate torn down) + // counted as an attempt would push messages toward the DLQ for + // something that never failed. Re-raised, it leaves the batch + // unacked and the platform redelivers it. + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.forEach(group, ({ message }) => retryOrExhaust(message, summarizeCause(cause)), { + discard: true, + }), ), ), { concurrency: 3, discard: true }, diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts index 428fac493..800d39bc2 100644 --- a/apps/api/src/routes/v2/audit-log.http.ts +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -13,7 +13,8 @@ import { } from "@maple/domain/http/v2" import type { V2AuditLogEntry } from "@maple/domain/http/v2" import type { AuditLogEntry } from "@/services/audit/audit-event" -import { Effect, Option, Schema } from "effect" +import { Cause, Effect, Option, Schema } from "effect" +import { summarizeCause } from "@/platform/describe-cause" import { AuditLogService } from "@/services/audit/AuditLogService" import { OrgMembersService } from "@/services/org/OrgMembersService" import { requireAdmin } from "@/services/auth/auth" @@ -22,10 +23,11 @@ import type { AuditLogListFilters } from "@/services/audit/AuditLogService" const adminOnly = () => V2InsufficientPermissions.make("Only org admins can read the audit log") /** No directory, no names — the entries still carry every id they were written with. */ -const unnamed = (cause: unknown) => - Effect.logWarning("Audit log: member directory unavailable; entries keep their ids", { - cause, - }).pipe(Effect.as(new Map())) +const unnamed = (cause: Cause.Cause) => + Effect.logWarning("Audit log: member directory unavailable; entries keep their ids").pipe( + Effect.annotateLogs({ error: summarizeCause(cause) }), + Effect.as(new Map()), + ) const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) @@ -164,8 +166,8 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( * actually recognises — an opaque `user_…` is the last resort, not the * second one. * - * `catch` + `catchDefect` rather than `catchCause`, which would also - * swallow the interrupt that tears this request down. + * Failures and defects both fall back; interruption is re-raised, because + * a request being torn down has no page left to label. */ const directory = (orgId: OrgId) => members.listMembers(orgId).pipe( @@ -181,8 +183,9 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( ), ), ), - Effect.catch((error) => unnamed(error)), - Effect.catchDefect((defect) => unnamed(defect)), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) ? Effect.interrupt : unnamed(cause), + ), ) return handlers.handle("list", ({ query }) => From 1fcdcb58a2769df96ccecafc8feb664f85b2f5ed Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 4 Sep 2026 18:07:54 +0200 Subject: [PATCH 18/19] refactor(worker): the runtime owns the flush; drop the try/finally around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every queue consumer and cron in worker.ts wrapped `runScheduledEffect` in a `try/finally` for one reason: to register the telemetry flush. Six copies of the same hand-rolled boundary around a helper that already owns the runtime, the scheduler drain, the dispose and the `waitUntil`. `runScheduledEffect` takes `onSettled` now and runs it after dispose, inside the `waitUntil` it already registered. That is also strictly safer than what the callers did: a `finally` that fires after the awaited promise settles was calling `ctx.waitUntil` late, sometimes after the handler had already rejected; chaining it into the existing registration cannot miss the window. And the flush still runs after dispose, where the last spans have been emitted. Converted all six — the two other queue consumers and the three crons alongside the audit one, since leaving worker.ts half-migrated would be worse than either end state. `onSettled` is typed `() => Promise`, not `Promise`: the repo's anti-slop lint rejects handing `unknown` back to a caller, and the SDK's flush is `Promise` anyway. apps/api 215 files / 2638 tests, infra 57, typecheck 40/40, lint clean. Co-Authored-By: Claude Opus 5 --- apps/api/src/worker.ts | 91 ++++++++----------- .../infra/src/cloudflare/worker-runtime.ts | 14 ++- 2 files changed, 49 insertions(+), 56 deletions(-) diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index 261f314ff..5eba84afa 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -417,26 +417,24 @@ const handleQueue = async ( processPlanetScaleWebhookBatch, flushPlanetScaleWebhookTelemetry, } = await import("./planetscale-webhook-runtime") - try { - await runScheduledEffect( - buildPlanetScaleWebhookLayer(env), - await scoped(processPlanetScaleWebhookBatch(batch)), - ctx, - ) - } finally { - ctx.waitUntil(flushPlanetScaleWebhookTelemetry(env)) - } + await runScheduledEffect( + buildPlanetScaleWebhookLayer(env), + await scoped(processPlanetScaleWebhookBatch(batch)), + ctx, + { onSettled: () => flushPlanetScaleWebhookTelemetry(env) }, + ) return } if (queueKind === "audit-events") { const { buildAuditEventsLayer, processAuditEventsBatch, flushAuditEventsTelemetry } = await import( "./audit-events-runtime" ) - try { - await runScheduledEffect(buildAuditEventsLayer(env), await scoped(processAuditEventsBatch(batch)), ctx) - } finally { - ctx.waitUntil(flushAuditEventsTelemetry(env)) - } + await runScheduledEffect( + buildAuditEventsLayer(env), + await scoped(processAuditEventsBatch(batch)), + ctx, + { onSettled: () => flushAuditEventsTelemetry(env) }, + ) return } if (queueKind === "unknown") { @@ -444,11 +442,9 @@ const handleQueue = async ( } const { buildVcsSyncLayer, processBatch, flushVcsTelemetry } = await import("./vcs-sync-runtime") - try { - await runScheduledEffect(buildVcsSyncLayer(env), await scoped(processBatch(batch)), ctx) - } finally { - ctx.waitUntil(flushVcsTelemetry(env)) - } + await runScheduledEffect(buildVcsSyncLayer(env), await scoped(processBatch(batch)), ctx, { + onSettled: () => flushVcsTelemetry(env), + }) } // Cron handler. Three schedules (see `crons` in alchemy.run.ts), dispatched on @@ -475,52 +471,37 @@ const handleScheduled = async ( const { runScrapeCheckRetention } = await import("@/services/integrations/scrape-check-retention") const { runPlanetScaleEventRetention } = await import("@/services/integrations/planetscale-event-retention") - try { - // Both sweeps ride this one cron: each new cron string costs an entry in - // alchemy.run.ts and a branch here, and neither needs its own beat. - // Sequential, not concurrent — they share one Postgres socket for the - // whole tick, so running them concurrently would only queue on it. - await runScheduledEffect( - buildScrapeRetentionLayer(env), - await scoped( - Effect.andThen(runScrapeCheckRetention, runPlanetScaleEventRetention), - ), - ctx, - { onInterrupt: "graceful" }, - ) - } finally { - ctx.waitUntil(flushVcsTelemetry(env)) - } + // Both sweeps ride this one cron: each new cron string costs an entry in + // alchemy.run.ts and a branch here, and neither needs its own beat. + // Sequential, not concurrent — they share one Postgres socket for the + // whole tick, so running them concurrently would only queue on it. + await runScheduledEffect( + buildScrapeRetentionLayer(env), + await scoped(Effect.andThen(runScrapeCheckRetention, runPlanetScaleEventRetention)), + ctx, + { onInterrupt: "graceful", onSettled: () => flushVcsTelemetry(env) }, + ) return } if (event.cron === SLACK_RECONCILE_CRON) { const { buildSlackReconcileLayer, runSlackReconciliation, flushSlackTelemetry } = await import("./slack-reconcile-runtime") - try { - await runScheduledEffect( - buildSlackReconcileLayer(env), - await scoped(runSlackReconciliation), - ctx, - { onInterrupt: "graceful" }, - ) - } finally { - ctx.waitUntil(flushSlackTelemetry(env)) - } + await runScheduledEffect(buildSlackReconcileLayer(env), await scoped(runSlackReconciliation), ctx, { + onInterrupt: "graceful", + onSettled: () => flushSlackTelemetry(env), + }) return } const { buildVcsScheduledLayer, runScheduledSync, flushVcsTelemetry } = await import("./vcs-sync-runtime") - try { - // Graceful on interrupt: a teardown mid-cron is expected lifecycle, and the - // schedule reruns — only the queue consumer above must keep rejecting so an - // interrupted batch redelivers instead of acking. - await runScheduledEffect(buildVcsScheduledLayer(env), await scoped(runScheduledSync), ctx, { - onInterrupt: "graceful", - }) - } finally { - ctx.waitUntil(flushVcsTelemetry(env)) - } + // Graceful on interrupt: a teardown mid-cron is expected lifecycle, and the + // schedule reruns — only the queue consumer above must keep rejecting so an + // interrupted batch redelivers instead of acking. + await runScheduledEffect(buildVcsScheduledLayer(env), await scoped(runScheduledSync), ctx, { + onInterrupt: "graceful", + onSettled: () => flushVcsTelemetry(env), + }) } /** diff --git a/packages/infra/src/cloudflare/worker-runtime.ts b/packages/infra/src/cloudflare/worker-runtime.ts index 65799bd06..b2bcbdb88 100644 --- a/packages/infra/src/cloudflare/worker-runtime.ts +++ b/packages/infra/src/cloudflare/worker-runtime.ts @@ -104,6 +104,12 @@ export const withRequestRuntime = , Ctx e * draining the scheduler first and registering the whole thing with * `ctx.waitUntil`. Rethrows so the CF runtime reports the failure. * + * `onSettled` runs once the runtime is disposed, inside the same `waitUntil` + * registration — for the telemetry flush every handler owes at the end of an + * invocation. Registering it here rather than in a caller's `finally` keeps it + * inside a `waitUntil` the platform has already accepted, and keeps the flush + * after dispose, where the last spans have been emitted. + * * `onInterrupt` decides what an interrupt-only exit (isolate teardown mid-run) * looks like to the caller: * - `"reject"` (default): rethrow, so the CF runtime reports the invocation as @@ -117,7 +123,10 @@ export const runScheduledEffect = ( layer: Layer.Layer, program: Effect.Effect, ctx: ExecutionContextLike, - options?: { readonly onInterrupt?: "reject" | "graceful" }, + options?: { + readonly onInterrupt?: "reject" | "graceful" + readonly onSettled?: () => Promise + }, ): Promise => { const runtime = ManagedRuntime.make(layer) const done = runtime @@ -135,6 +144,9 @@ export const runScheduledEffect = ( await runtime.dispose().catch((err) => { console.error("[worker-runtime] scheduled runtime dispose failed:", err) }) + await options?.onSettled?.().catch((err) => { + console.error("[worker-runtime] scheduled onSettled failed:", err) + }) }) ctx.waitUntil(done.catch(() => undefined)) return done From 68ea5084434d765542e517911204bb29ec9baa25 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 5 Sep 2026 11:06:33 +0200 Subject: [PATCH 19/19] feat(web): audit log rows open into their full record Every entry is now a button that expands an inline detail panel: occurred and recorded times, the actor with its full id, source and origin, outcome with the denial reason, resource and request ids with copy affordances, the entry's public alog_ id, a before/after table for updates, and the recorded metadata with request bodies, tool parameters and SQL pretty-printed as blocks. The native title tooltips that carried this before were unreachable by keyboard and unreadable for anything longer than a line. The list itself: the verb carries the weight in the action column, a user the directory could not name shows an abbreviated id instead of the whole string, a system entry no longer prints a dash for a name, membership events show the affected user as the resource, the header aligns over the timestamps, filters report an empty match with a way to clear them, a refresh re-reads the first page, and filter tabs expose aria-pressed. Lanes now collapse on the card's own width (@container) instead of the viewport: the sidebar and settings nav leave a 1440px window with an ~890px card, and the viewport breakpoints were overflowing it horizontally. Below @md a row wraps the action onto its own line so the phone layout stops pushing it off-screen. --- .../components/settings/audit-log-section.tsx | 569 ++++++++++++++---- 1 file changed, 454 insertions(+), 115 deletions(-) diff --git a/apps/web/src/components/settings/audit-log-section.tsx b/apps/web/src/components/settings/audit-log-section.tsx index 43be44a53..da8eb40b9 100644 --- a/apps/web/src/components/settings/audit-log-section.tsx +++ b/apps/web/src/components/settings/audit-log-section.tsx @@ -1,5 +1,11 @@ import type { AuditActorType, AuditOutcome } from "@maple/domain/http" -import type { V2AuditChanges, V2AuditLogEntry } from "@maple/domain/http/v2" +import { + encodePublicId, + PublicIdPrefixes, + type V2AuditChanges, + type V2AuditLogEntry, +} from "@maple/domain/http/v2" +import { Option } from "effect" import { useState, type ReactNode } from "react" import { Result, useAtomRefresh, useAtomValue } from "@/lib/effect-atom" @@ -8,11 +14,13 @@ import { auditLogPageAtom } from "@/lib/services/atoms/audit-log-atoms" import { Avatar, AvatarFallback, AvatarImage } from "@maple/ui/components/ui/avatar" import { Badge } from "@maple/ui/components/ui/badge" import { Button } from "@maple/ui/components/ui/button" +import { CopyButton } from "@maple/ui/components/ui/copy-button" import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { trySync } from "@maple/ui/lib/try-sync" import { cn } from "@maple/ui/lib/utils" import { formatRelativeTime } from "@maple/ui/lib/time-format" -import { AlertWarningIcon, HistoryIcon } from "@/components/icons" +import { AlertWarningIcon, ArrowPathIcon, ChevronRightIcon, HistoryIcon } from "@/components/icons" type ActorFilter = AuditActorType | "all" type OutcomeFilter = AuditOutcome | "all" @@ -39,13 +47,19 @@ const ACTOR_BADGES: Record // Shared column lanes so the header row and entry rows stay aligned. Resource and -// source collapse on narrower viewports; time + actor + action always stay visible. +// source collapse when the card is narrow; time + actor + action always stay +// visible, and action is the lane that absorbs the remaining width. The card, +// not the viewport, is what the breakpoints measure: the sidebar and settings +// nav leave it far narrower than the window. +// Below `@md` a row wraps: time + actor on the first line, the action on its own +// line beneath, indented past the chevron. const COL = { - time: "w-[96px] shrink-0", - actor: "w-[200px] min-w-0 shrink-0", - action: "min-w-0 flex-1", - resource: "hidden w-[220px] min-w-0 shrink-0 md:block", - source: "hidden w-[80px] shrink-0 lg:block", + time: "flex w-[112px] shrink-0 items-center gap-2", + actor: "min-w-0 flex-1 @md:w-[176px] @md:flex-none", + action: "min-w-0 basis-full pl-5 @md:basis-0 @md:flex-1 @md:pl-0", + resource: "hidden w-[200px] min-w-0 shrink-0 @3xl:block", + // 52rem: the card's width at a 1440px window, the narrowest desktop that should still show it. + source: "hidden w-[104px] shrink-0 @min-[52rem]:block", } const COL_HEADER = "text-muted-foreground/70 font-mono text-[10px] uppercase tracking-[0.12em]" @@ -66,6 +80,23 @@ function initialsOf(label: string): string { .join("") } +/** + * `user_3BfcmIS3bUNV6BfAEkR2WzFOCvu` → `user_…FOCvu`. The prefix says what kind + * of id it is and the tail is what someone compares against a copied value; + * the middle is noise in a 200px lane. The full id stays in the detail panel. + */ +function abbreviateId(id: string): string { + const prefixEnd = id.indexOf("_") + if (prefixEnd === -1 || id.length <= prefixEnd + 10) return id + return `${id.slice(0, prefixEnd + 1)}…${id.slice(-5)}` +} + +function actorDisplayName(entry: V2AuditLogEntry): string | null { + if (entry.actor_name !== null) return entry.actor_name + if (entry.actor_id !== null) return abbreviateId(entry.actor_id) + return null +} + function formatDateTime(value: string): string { return new Date(value).toLocaleString(undefined, { month: "short", @@ -76,28 +107,49 @@ function formatDateTime(value: string): string { }) } -// JSON.stringify(undefined) is undefined — surface it as text in tooltips. -function formatChangeValue(value: unknown): string { - return JSON.stringify(value) ?? "undefined" +function formatDateTimeFull(value: string): string { + return new Date(value).toLocaleString(undefined, { + year: "numeric", + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + timeZoneName: "short", + }) } -function formatChangesTooltip(changes: V2AuditChanges): string { - return changes.fields - .map( - (field) => - `${field}: ${formatChangeValue(changes.before[field])} → ${formatChangeValue(changes.after[field])}`, - ) - .join("\n") +/** `` / `` are the audit pipeline's placeholders, not values. */ +function isPlaceholder(value: unknown): value is string { + return typeof value === "string" && /^<[a-z]+>$/.test(value) } -function formatSourceTooltip(entry: V2AuditLogEntry): string | undefined { - const lines = [ - entry.origin_ip !== null || entry.origin_country !== null - ? `From ${entry.origin_ip ?? "unknown IP"}${entry.origin_country !== null ? ` (${entry.origin_country})` : ""}` - : null, - entry.request_id !== null ? `Request ${entry.request_id}` : null, - ].filter((line) => line !== null) - return lines.length > 0 ? lines.join("\n") : undefined +// JSON.stringify(undefined) is undefined — surface it as text. +function formatScalar(value: unknown): string { + if (typeof value === "string") return value + return JSON.stringify(value) ?? "undefined" +} + +/** What a metadata string that opens with `{` or `[` parses to, when it parses at all. */ +type JsonDocument = Record | ReadonlyArray + +/** + * Metadata values are stored as they were recorded: request bodies and tool + * parameters arrive as JSON text, SQL as a statement. Anything structured, or + * long enough to wrap, is rendered as a block rather than inline. + */ +function metadataBlock(value: unknown): string | null { + if (typeof value === "string") { + const trimmed = value.trim() + const parsed = + trimmed.startsWith("{") || trimmed.startsWith("[") + ? Option.getOrNull(trySync((): JsonDocument => JSON.parse(trimmed))) + : null + if (parsed !== null) return JSON.stringify(parsed, null, 2) + return value.length > 72 || value.includes("\n") ? value : null + } + if (value !== null && typeof value === "object") return JSON.stringify(value, null, 2) + return null } /** @@ -129,14 +181,21 @@ export function AuditLogSection() { // would otherwise shift later pages and make them repeat and skip rows. const [until, setUntil] = useState(undefined) + const filterInput = { + ...(actorFilter !== "all" ? { actorType: actorFilter } : undefined), + ...(outcomeFilter !== "all" ? { outcome: outcomeFilter } : undefined), + } const pageAtom = auditLogPageAtom({ + ...filterInput, ...(cursor !== undefined ? { cursor } : undefined), ...(until !== undefined ? { until } : undefined), - ...(actorFilter !== "all" ? { actorType: actorFilter } : undefined), - ...(outcomeFilter !== "all" ? { outcome: outcomeFilter } : undefined), }) + // The first page for the current filters — what Refresh re-fetches. Without + // the refresh the page family would hand back its cached copy from before. + const firstPageAtom = auditLogPageAtom(filterInput) const pageResult = useAtomValue(pageAtom) const refreshPage = useAtomRefresh(pageAtom) + const refreshFirstPage = useAtomRefresh(firstPageAtom) // Each Load more / filter change swaps to a new page atom, which starts in its // initial state. Keep the accumulated entries so the table stays rendered @@ -154,24 +213,44 @@ export function AuditLogSection() { }) } + function restartList() { + setCursor(undefined) + setUntil(undefined) + } + function handleFilterSelect(value: ActorFilter) { if (value === actorFilter) return setActorFilter(value) - setCursor(undefined) - setUntil(undefined) + restartList() } function handleOutcomeSelect(value: OutcomeFilter) { if (value === outcomeFilter) return setOutcomeFilter(value) - setCursor(undefined) - setUntil(undefined) + restartList() + } + + function clearFilters() { + setActorFilter("all") + setOutcomeFilter("all") + restartList() + } + + function handleRefresh() { + restartList() + refreshFirstPage() } const waiting = !Result.isSuccess(pageResult) || pageResult.waiting + const filtered = actorFilter !== "all" || outcomeFilter !== "all" return (
+

+ Changes, refused attempts, and every read of telemetry or session replays — from the dashboard, + API, and MCP. Select an entry for its full record. +

+
{ACTOR_FILTERS.map((filter) => ( @@ -196,13 +275,25 @@ export function AuditLogSection() { ))}
-

- Changes, refused attempts, and every read of telemetry or session replays — from the - dashboard, API, and MCP. -

+
-
+
{view === null && Result.isFailure(pageResult) ? ( @@ -224,6 +315,21 @@ export function AuditLogSection() {
+ ) : view.entries.length === 0 && filtered ? ( + + + + + + No entries match these filters + + Nothing recorded for this actor type and outcome. + + + + ) : view.entries.length === 0 ? ( @@ -238,8 +344,12 @@ export function AuditLogSection() { ) : (
-
- Time + -
- {entry.resource_type !== null || entry.resource_id !== null ? ( + > + + + {formatRelativeTime(entry.occurred_at)} + + +
- {entry.resource_type !== null && ( - - {entry.resource_type} + + {denied && ( + + Denied )} - {entry.resource_id !== null && ( - - {entry.resource_id} +
+ {denied && entry.denial_reason !== null && ( +

+ {entry.denial_reason} +

+ )} + {entry.changes !== null && entry.changes.fields.length > 0 && ( +

+ {entry.changes.fields.join(", ")} +

+ )} +
+
+ +
+ + {entry.source} + {entry.origin_country !== null && ( + · {entry.origin_country} + )} + + + {expanded && ( +
+ +
+ )} +
+ ) +} + +function ResourceCell({ entry }: { entry: V2AuditLogEntry }) { + // Membership changes act on a person: the affected user is the resource. + const id = entry.resource_id ?? entry.affected_user + if (entry.resource_type === null && id === null) { + return — + } + return ( +
+ {entry.resource_type !== null && ( + + {entry.resource_type} + + )} + {id !== null && ( + + {id} + + )} +
+ ) +} + +function DetailField({ label, children }: { label: string; children: ReactNode }) { + return ( + <> +
{label}
+
{children}
+ + ) +} + +/** An identifier with its copy affordance; the one place the full value is shown untruncated. */ +function Identifier({ value, label }: { value: string; label: string }) { + return ( + + {value} + + + ) +} + +function ChangeValue({ value }: { value: unknown }) { + if (isPlaceholder(value)) { + return {value.slice(1, -1)} + } + if (value === undefined || value === null || value === "") { + return — + } + return {formatScalar(value)} +} + +function ChangesTable({ changes }: { changes: V2AuditChanges }) { + return ( +
+ + + + + + + + + + {changes.fields.map((field) => ( + + + + + + ))} + +
FieldBeforeAfter
{field} + + + +
+
+ ) +} + +function MetadataList({ metadata }: { metadata: Record }) { + const keys = Object.keys(metadata) + if (keys.length === 0) return null + return ( +
+ {keys.map((key) => { + const value = metadata[key] + const block = metadataBlock(value) + return ( + + {block !== null ? ( + 
+								{block}
+
+ ) : ( + + )} -
- ) : ( - — + + ) + })} + + ) +} + +function AuditLogDetail({ entry }: { entry: V2AuditLogEntry }) { + const badge = ACTOR_BADGES[entry.actor_type] + const hasChanges = entry.changes !== null && entry.changes.fields.length > 0 + const hasMetadata = entry.metadata !== null && Object.keys(entry.metadata).length > 0 + + return ( +
+
+ + {formatDateTimeFull(entry.occurred_at)} + + + {formatDateTimeFull(entry.recorded_at)} + + + + + {badge.label} + + {entry.actor_name !== null && {entry.actor_name}} + {entry.actor_id !== null && } + + + + {entry.source} + {(entry.origin_ip !== null || entry.origin_country !== null) && ( + + {" · "} + {entry.origin_ip ?? "unknown IP"} + {entry.origin_country !== null && ` (${entry.origin_country})`} + + )} + + + + {entry.action} + {entry.outcome === "denied" ? ( + + Denied + + ) : ( + + Allowed + + )} + + {entry.denial_reason !== null && ( +

{entry.denial_reason}

+ )} + + + {entry.resource_type === null && entry.resource_id === null ? ( + — + ) : ( + + {entry.resource_type !== null && ( + + {entry.resource_type} + + )} + {entry.resource_id !== null && ( + + )} + + )} + + {entry.affected_user !== null && ( + + + )} -
- - {entry.source} - {entry.origin_country !== null && ( - · {entry.origin_country} + {entry.request_id !== null && ( + + + )} - + + {/* The wire codec hands the client the raw id; show the `alog_…` + form the API itself returns, so it can be quoted back to it. */} + + + + + {hasChanges && entry.changes !== null && ( +
+

Changes

+ +
+ )} + + {hasMetadata && entry.metadata !== null && ( +
+

Details

+ +
+ )}
) }