fix(edge): stop agent-edge 404ing every real /api/* route on GET

.fleet/evidence/landing-audit/scorecard.json

@@ -41,7 +41,13 @@
   },
   "geo": {
     "productionAgentIndex": "S 100%",
-    "surfaces": ["/llms.txt", "/llms-full.txt", "/index.md", "/api/ai", "/skill.md"]
+    "surfaces": [
+      "/llms.txt",
+      "/llms-full.txt",
+      "/index.md",
+      "/api/ai",
+      "/skill.md"
+    ]
   },
   "performance": {
     "tool": "PSI Swarm 0.4.2",

agent-edge.mjs

@@ -2,10 +2,16 @@
  * Portable agent-edge handler — copy or generate into each product.
  * Spec: fleet-ops/docs/agent-indexing-standard.md
  *
- * Usage in worker.mjs (before openNext.fetch):
- *   import { handleAgentEdge } from './agent-edge.mjs'
+ * The edge is a *strict pre-handler*: it answers only for the surfaces it
+ * genuinely owns and returns `null` for everything else. It must never decide
+ * that a path does not exist — only Next.js knows the route table.
+ *
+ * Usage in worker.mjs:
+ *   import { handleAgentEdge, withApiJsonNotFound } from './agent-edge.mjs'
  *   const agent = handleAgentEdge(request)
  *   if (agent) return agent
+ *   // ...and on the way back out, so unknown /api/* paths answer in JSON:
+ *   return withApiJsonNotFound(request, await openNext.fetch(request, env, ctx))
  */
 
 import {
@@ -574,6 +580,15 @@ function catalogFor(origin) {
 }
 
 /**
+ * Pre-handler: answers only for surfaces the edge itself owns, and returns
+ * `null` for everything else so the request reaches OpenNext/Next.js.
+ *
+ * The allow-list (`EXACT_ROUTES`) is additive — it *grants* the edge specific
+ * paths. There is deliberately no subtractive `/api/*` catch-all here: the edge
+ * does not know the Next.js route table and must never answer 404 on its
+ * behalf. Unknown `/api/*` paths are shaped into JSON by `withApiJsonNotFound`
+ * on the way back out instead.
+ *
  * @param {Request} request
  * @returns {Response | null}
  */
@@ -586,10 +601,10 @@ export function handleAgentEdge(request) {
   const exactResponse = exact ? exact(url) : null;
   if (exactResponse) return exactResponse;
 
-  // JSON errors for unknown /api/* paths.
-  if (path.startsWith('/api/')) {
-    return jsonError(404, 'not_found', `Unknown API path: ${path}`, path);
-  }
+  // `/api/*` belongs to Next.js route handlers. Fall through unconditionally —
+  // an Accept header must never divert a real API request away from its
+  // handler, and the edge must never claim the path is missing.
+  if (isApiPath(path)) return null;
 
   if (!wantsMarkdown(request)) return null;
 
@@ -607,6 +622,35 @@ export function handleAgentEdge(request) {
   return null;
 }
 
+/**
+ * Post-handler: shape Next.js's own 404 for `/api/*` into a JSON error body.
+ *
+ * The edge deliberately does not know which API routes exist — Next.js does.
+ * We call it, and only if *it* reports 404 do we swap the HTML error page for
+ * the machine-readable JSON envelope. That is why this shape cannot rot the way
+ * the previous `/api/*` catch-all did: a route handler added under
+ * `src/app/api/` is reachable with no edge change, because the edge never
+ * asserts non-existence.
+ *
+ * @param {Request} request
+ * @param {Response} response Response from the downstream Next.js handler.
+ * @returns {Response}
+ */
+export function withApiJsonNotFound(request, response) {
+  if (request.method !== 'GET' && request.method !== 'HEAD') return response;
+  if (response.status !== 404) return response;
+  const path = new URL(request.url).pathname;
+  if (!isApiPath(path)) return response;
+  const contentType = response.headers.get('content-type') || '';
+  // A route handler's own JSON 404 is already machine-readable — leave it.
+  if (contentType.includes('application/json')) return response;
+  return jsonError(404, 'not_found', `Unknown API path: ${path}`, path);
+}
+
+function isApiPath(pathname) {
+  return pathname.startsWith('/api/');
+}
+
 /**
  * Exact-path agent surfaces. Each handler takes the request URL so it can
  * bind responses to the origin that was asked for.

docs/architecture/edge-worker.md

@@ -28,6 +28,31 @@ live in `worker.mjs` / `worker-routing.mjs` / `agent-edge.mjs` instead.
 | `rate-limiter-do.mjs` | `RateLimiterDO` Durable Object backing `src/lib/rate-limit.ts`. |
 | `timing.mjs` | `withTiming()` wrapper for request timing. |
 
+## The edge never owns `/api/*` it did not declare
+
+`agent-edge.mjs` is a **strict pre-handler**. It answers only for the exact
+paths in its `EXACT_ROUTES` allow-list (`/llms.txt`, `/llms-full.txt`,
+`/index.md`, `/robots.txt`, `/openapi.json`, `/openapi.yaml`, `/api/ai`) and
+returns `null` for everything else, so the request reaches OpenNext. It must
+never conclude that a path does *not* exist — only Next.js knows the route
+table.
+
+The JSON error envelope for unknown API paths is a **post-handler**,
+`withApiJsonNotFound(request, response)`, applied in `worker.mjs` to the
+response coming back from OpenNext: only when Next.js itself answers 404 for an
+`/api/*` path does the HTML error page become
+`{"error":{"code":"not_found",…}}`. A route handler's own JSON 404 is left
+untouched.
+
+> **Carry-forward when regenerating `agent-edge.mjs`.** This file is a
+> copied/generated fleet artifact (see its header and the
+> `apply-agent-surfaces` payload marker). The upstream template's `/api/*`
+> catch-all 404 shadowed every real Next.js route handler in production —
+> `GET /api/pages` returned the edge's `not_found` envelope while `POST` to the
+> same path reached its handler. Regenerating from an unfixed template
+> reintroduces the outage. `tests/agent-edge-api-routing.unit.test.mjs` fails
+> loudly if it comes back; do not "fix" it by narrowing the test.
+
 ## Cacheable document paths
 
 `worker.mjs` keeps a `CACHEABLE_EXACT` set of landing/marketing document paths

docs/development/conventions.md

@@ -31,6 +31,9 @@ React Compiler is **ON** (`babel-plugin-react-compiler`).
 - `middleware.ts` / `proxy.ts` for edge guards — not supported by the
   Cloudflare OpenNext adapter. Use `worker.mjs` / `worker-routing.mjs` /
   `agent-edge.mjs` (ADR 0001).
+- A subtractive `/api/*` catch-all in `agent-edge.mjs`. It 404'd every real
+  Next.js route handler on GET in production. See the carry-forward note in
+  `docs/architecture/edge-worker.md`.
 - SaaS Maker RAG as a profile-memory fallback — removed; only the shared
   `knowledgebase` Worker is used (`docs/architecture/rag-memory.md`).
 - The legacy `unsafe` native ratelimit binding — replaced by `RateLimiterDO`

knip.json

@@ -22,6 +22,7 @@
     "**/eslint.config.*",
     "scripts/**",
     "tests/e2e/**",
+    "tests/fixtures/**",
     "test/**",
     "**/*.test.{ts,tsx,js,mjs}",
     "**/*.spec.{ts,tsx,js,mjs}",

tests/agent-edge-api-routing.unit.test.mjs

@@ -0,0 +1,220 @@
+/**
+ * Regression guard for the edge shadowing real API routes.
+ *
+ * `agent-edge.mjs` carried a `path.startsWith('/api/')` catch-all that answered
+ * 404 *before* OpenNext ever saw the request, with only `/api/ai` allow-listed
+ * above it. Every other `GET /api/*` — `/api/pages`, `/api/v1/agents`, the
+ * whole better-auth surface — was 404'd at the edge in production, while POST
+ * to the same paths reached its handler.
+ *
+ * These tests drive the real `worker.mjs` entrypoint with a stubbed OpenNext
+ * handler (see `fixtures/open-next-worker-stub.mjs`), so they cover the actual
+ * wiring, not a reimplementation of it.
+ *
+ * The route list is read off the filesystem rather than hard-coded: a route
+ * handler added under `src/app/api/` is covered the moment it lands, which is
+ * exactly the rot the original hand-maintained allow-list suffered.
+ */
+import { readdirSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import worker from '../worker.mjs';
+import { openNextStub } from './fixtures/open-next-worker-stub.mjs';
+
+const API_DIR = resolve(
+  dirname(fileURLToPath(import.meta.url)),
+  '../src/app/api',
+);
+
+/** Turn `src/app/api/pages/[pageId]/links/route.ts` into `/api/pages/sample/links`. */
+function collectApiRoutePaths(dir, prefix = '/api') {
+  const paths = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.isDirectory()) {
+      const dynamic = entry.name.startsWith('[');
+      const catchAll =
+        entry.name.startsWith('[...') || entry.name.startsWith('[[...');
+      const segment = dynamic
+        ? catchAll
+          ? 'sample/segment'
+          : 'sample'
+        : entry.name;
+      paths.push(
+        ...collectApiRoutePaths(join(dir, entry.name), `${prefix}/${segment}`),
+      );
+    } else if (entry.name === 'route.ts' || entry.name === 'route.tsx') {
+      paths.push(prefix);
+    }
+  }
+  return paths;
+}
+
+const API_ROUTE_PATHS = collectApiRoutePaths(API_DIR);
+
+const ctx = {
+  waitUntil: () => undefined,
+  passThroughOnException: () => undefined,
+};
+const env = { NEXT_PUBLIC_APP_URL: 'https://karte.cc' };
+
+function request(path, { method = 'GET', headers = {} } = {}) {
+  return worker.fetch(
+    new Request(`https://karte.cc${path}`, {
+      method,
+      headers: { host: 'karte.cc', ...headers },
+    }),
+    env,
+    ctx,
+  );
+}
+
+beforeEach(() => {
+  openNextStub.reset();
+});
+
+describe('edge does not shadow Next.js API routes', () => {
+  it('found the route handlers to guard', () => {
+    // Sanity check: if this ever hits zero the suite below is vacuous.
+    expect(API_ROUTE_PATHS.length).toBeGreaterThan(20);
+    expect(API_ROUTE_PATHS).toContain('/api/pages');
+    expect(API_ROUTE_PATHS).toContain('/api/auth/sample/segment');
+  });
+
+  it('reaches a real API route through the edge on GET', async () => {
+    openNextStub.handler = () =>
+      new Response(JSON.stringify({ ok: true }), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      });
+
+    const response = await request('/api/pages');
+
+    expect(openNextStub.calls).toEqual(['/api/pages']);
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ ok: true });
+  });
+
+  it.each(API_ROUTE_PATHS)('lets GET %s reach Next.js', async (path) => {
+    openNextStub.handler = () => new Response('routed', { status: 200 });
+
+    const response = await request(path);
+
+    expect(openNextStub.calls).toEqual([path]);
+    expect(response.status).toBe(200);
+  });
+
+  it('lets HEAD reach Next.js too', async () => {
+    openNextStub.handler = () => new Response(null, { status: 200 });
+
+    const response = await request('/api/pages', { method: 'HEAD' });
+
+    expect(openNextStub.calls).toEqual(['/api/pages']);
+    expect(response.status).toBe(200);
+  });
+
+  it('does not let an Accept header divert an API route to the markdown 404', async () => {
+    openNextStub.handler = () => new Response('routed', { status: 200 });
+
+    const response = await request('/api/pages', {
+      headers: { accept: 'text/markdown' },
+    });
+
+    expect(openNextStub.calls).toEqual(['/api/pages']);
+    expect(response.status).toBe(200);
+  });
+});
+
+describe('unknown API paths still answer with the JSON 404 envelope', () => {
+  it('replaces the Next.js HTML 404 with JSON', async () => {
+    openNextStub.handler = () =>
+      new Response('<!DOCTYPE html><html><body>404</body></html>', {
+        status: 404,
+        headers: { 'content-type': 'text/html; charset=utf-8' },
+      });
+
+    const response = await request('/api/definitely-not-a-real-path');
+
+    expect(openNextStub.calls).toEqual(['/api/definitely-not-a-real-path']);
+    expect(response.status).toBe(404);
+    expect(response.headers.get('content-type')).toContain('application/json');
+    expect(await response.json()).toEqual({
+      error: {
+        code: 'not_found',
+        message: 'Unknown API path: /api/definitely-not-a-real-path',
+        path: '/api/definitely-not-a-real-path',
+      },
+    });
+  });
+
+  it("leaves a route handler's own JSON 404 body untouched", async () => {
+    openNextStub.handler = () =>
+      new Response(JSON.stringify({ error: 'page not found' }), {
+        status: 404,
+        headers: { 'content-type': 'application/json' },
+      });
+
+    const response = await request('/api/pages/999999/links');
+
+    expect(await response.json()).toEqual({ error: 'page not found' });
+  });
+
+  it('leaves non-API 404s alone so the HTML error page still renders', async () => {
+    openNextStub.handler = () =>
+      new Response('<!DOCTYPE html><html><body>404</body></html>', {
+        status: 404,
+        headers: { 'content-type': 'text/html; charset=utf-8' },
+      });
+
+    const response = await request('/no-such-page');
+
+    expect(openNextStub.calls).toEqual(['/no-such-page']);
+    expect(response.status).toBe(404);
+    expect(response.headers.get('content-type')).toContain('text/html');
+  });
+});
+
+describe('edge-owned surfaces still work', () => {
+  it('serves the agent catalog at /api/ai without touching Next.js', async () => {
+    const response = await request('/api/ai');
+
+    expect(openNextStub.calls).toEqual([]);
+    expect(response.status).toBe(200);
+    const catalog = await response.json();
+    expect(catalog.name).toBe('Karte');
+    expect(catalog.url).toBe('https://karte.cc');
+  });
+
+  it('serves llms.txt, llms-full.txt, index.md, robots.txt and openapi.json', async () => {
+    for (const path of [
+      '/llms.txt',
+      '/llms-full.txt',
+      '/index.md',
+      '/robots.txt',
+      '/openapi.json',
+    ]) {
+      expect((await request(path)).status, path).toBe(200);
+    }
+    expect(openNextStub.calls).toEqual([]);
+  });
+
+  it('still negotiates markdown on the homepage', async () => {
+    const response = await request('/', {
+      headers: { accept: 'text/markdown' },
+    });
+
+    expect(openNextStub.calls).toEqual([]);
+    expect(response.headers.get('content-type')).toContain('text/markdown');
+  });
+
+  it('still serves the markdown 404 for unknown non-API pages', async () => {
+    const response = await request('/no-such-page', {
+      headers: { accept: 'text/markdown' },
+    });
+
+    expect(openNextStub.calls).toEqual([]);
+    expect(response.status).toBe(404);
+    expect(response.headers.get('content-type')).toContain('text/markdown');
+  });
+});

tests/fixtures/cloudflare-workers-stub.mjs

@@ -0,0 +1,11 @@
+/**
+ * Stand-in for the `cloudflare:workers` runtime module, which only exists
+ * inside workerd. `rate-limiter-do.mjs` imports `DurableObject` from it and is
+ * pulled in transitively by `worker.mjs`, the entrypoint under test.
+ */
+export class DurableObject {
+  constructor(ctx, env) {
+    this.ctx = ctx;
+    this.env = env;
+  }
+}