diff --git a/.fleet/evidence/landing-audit/scorecard.json b/.fleet/evidence/landing-audit/scorecard.json index e3a7869..d97fd55 100644 --- a/.fleet/evidence/landing-audit/scorecard.json +++ b/.fleet/evidence/landing-audit/scorecard.json @@ -41,7 +41,13 @@ }, "geo": { "productionAgentIndex": "S 100%", - "surfaces": ["/llms.txt", "/llms-full.txt", "/index.md", "/api/ai", "/skill.md"] + "surfaces": [ + "/llms.txt", + "/llms-full.txt", + "/index.md", + "/api/ai", + "/skill.md" + ] }, "performance": { "tool": "PSI Swarm 0.4.2", diff --git a/agent-edge.mjs b/agent-edge.mjs index 365794a..4ed4b2e 100644 --- a/agent-edge.mjs +++ b/agent-edge.mjs @@ -2,10 +2,16 @@ * Portable agent-edge handler — copy or generate into each product. * Spec: fleet-ops/docs/agent-indexing-standard.md * - * Usage in worker.mjs (before openNext.fetch): - * import { handleAgentEdge } from './agent-edge.mjs' + * The edge is a *strict pre-handler*: it answers only for the surfaces it + * genuinely owns and returns `null` for everything else. It must never decide + * that a path does not exist — only Next.js knows the route table. + * + * Usage in worker.mjs: + * import { handleAgentEdge, withApiJsonNotFound } from './agent-edge.mjs' * const agent = handleAgentEdge(request) * if (agent) return agent + * // ...and on the way back out, so unknown /api/* paths answer in JSON: + * return withApiJsonNotFound(request, await openNext.fetch(request, env, ctx)) */ import { @@ -574,6 +580,15 @@ function catalogFor(origin) { } /** + * Pre-handler: answers only for surfaces the edge itself owns, and returns + * `null` for everything else so the request reaches OpenNext/Next.js. + * + * The allow-list (`EXACT_ROUTES`) is additive — it *grants* the edge specific + * paths. There is deliberately no subtractive `/api/*` catch-all here: the edge + * does not know the Next.js route table and must never answer 404 on its + * behalf. Unknown `/api/*` paths are shaped into JSON by `withApiJsonNotFound` + * on the way back out instead. + * * @param {Request} request * @returns {Response | null} */ @@ -586,10 +601,10 @@ export function handleAgentEdge(request) { const exactResponse = exact ? exact(url) : null; if (exactResponse) return exactResponse; - // JSON errors for unknown /api/* paths. - if (path.startsWith('/api/')) { - return jsonError(404, 'not_found', `Unknown API path: ${path}`, path); - } + // `/api/*` belongs to Next.js route handlers. Fall through unconditionally — + // an Accept header must never divert a real API request away from its + // handler, and the edge must never claim the path is missing. + if (isApiPath(path)) return null; if (!wantsMarkdown(request)) return null; @@ -607,6 +622,35 @@ export function handleAgentEdge(request) { return null; } +/** + * Post-handler: shape Next.js's own 404 for `/api/*` into a JSON error body. + * + * The edge deliberately does not know which API routes exist — Next.js does. + * We call it, and only if *it* reports 404 do we swap the HTML error page for + * the machine-readable JSON envelope. That is why this shape cannot rot the way + * the previous `/api/*` catch-all did: a route handler added under + * `src/app/api/` is reachable with no edge change, because the edge never + * asserts non-existence. + * + * @param {Request} request + * @param {Response} response Response from the downstream Next.js handler. + * @returns {Response} + */ +export function withApiJsonNotFound(request, response) { + if (request.method !== 'GET' && request.method !== 'HEAD') return response; + if (response.status !== 404) return response; + const path = new URL(request.url).pathname; + if (!isApiPath(path)) return response; + const contentType = response.headers.get('content-type') || ''; + // A route handler's own JSON 404 is already machine-readable — leave it. + if (contentType.includes('application/json')) return response; + return jsonError(404, 'not_found', `Unknown API path: ${path}`, path); +} + +function isApiPath(pathname) { + return pathname.startsWith('/api/'); +} + /** * Exact-path agent surfaces. Each handler takes the request URL so it can * bind responses to the origin that was asked for. diff --git a/docs/architecture/edge-worker.md b/docs/architecture/edge-worker.md index c451eb2..ce62225 100644 --- a/docs/architecture/edge-worker.md +++ b/docs/architecture/edge-worker.md @@ -28,6 +28,31 @@ live in `worker.mjs` / `worker-routing.mjs` / `agent-edge.mjs` instead. | `rate-limiter-do.mjs` | `RateLimiterDO` Durable Object backing `src/lib/rate-limit.ts`. | | `timing.mjs` | `withTiming()` wrapper for request timing. | +## The edge never owns `/api/*` it did not declare + +`agent-edge.mjs` is a **strict pre-handler**. It answers only for the exact +paths in its `EXACT_ROUTES` allow-list (`/llms.txt`, `/llms-full.txt`, +`/index.md`, `/robots.txt`, `/openapi.json`, `/openapi.yaml`, `/api/ai`) and +returns `null` for everything else, so the request reaches OpenNext. It must +never conclude that a path does *not* exist — only Next.js knows the route +table. + +The JSON error envelope for unknown API paths is a **post-handler**, +`withApiJsonNotFound(request, response)`, applied in `worker.mjs` to the +response coming back from OpenNext: only when Next.js itself answers 404 for an +`/api/*` path does the HTML error page become +`{"error":{"code":"not_found",…}}`. A route handler's own JSON 404 is left +untouched. + +> **Carry-forward when regenerating `agent-edge.mjs`.** This file is a +> copied/generated fleet artifact (see its header and the +> `apply-agent-surfaces` payload marker). The upstream template's `/api/*` +> catch-all 404 shadowed every real Next.js route handler in production — +> `GET /api/pages` returned the edge's `not_found` envelope while `POST` to the +> same path reached its handler. Regenerating from an unfixed template +> reintroduces the outage. `tests/agent-edge-api-routing.unit.test.mjs` fails +> loudly if it comes back; do not "fix" it by narrowing the test. + ## Cacheable document paths `worker.mjs` keeps a `CACHEABLE_EXACT` set of landing/marketing document paths diff --git a/docs/development/conventions.md b/docs/development/conventions.md index 5823822..29cf573 100644 --- a/docs/development/conventions.md +++ b/docs/development/conventions.md @@ -31,6 +31,9 @@ React Compiler is **ON** (`babel-plugin-react-compiler`). - `middleware.ts` / `proxy.ts` for edge guards — not supported by the Cloudflare OpenNext adapter. Use `worker.mjs` / `worker-routing.mjs` / `agent-edge.mjs` (ADR 0001). +- A subtractive `/api/*` catch-all in `agent-edge.mjs`. It 404'd every real + Next.js route handler on GET in production. See the carry-forward note in + `docs/architecture/edge-worker.md`. - SaaS Maker RAG as a profile-memory fallback — removed; only the shared `knowledgebase` Worker is used (`docs/architecture/rag-memory.md`). - The legacy `unsafe` native ratelimit binding — replaced by `RateLimiterDO` diff --git a/knip.json b/knip.json index bf7fc80..216629e 100644 --- a/knip.json +++ b/knip.json @@ -22,6 +22,7 @@ "**/eslint.config.*", "scripts/**", "tests/e2e/**", + "tests/fixtures/**", "test/**", "**/*.test.{ts,tsx,js,mjs}", "**/*.spec.{ts,tsx,js,mjs}", diff --git a/tests/agent-edge-api-routing.unit.test.mjs b/tests/agent-edge-api-routing.unit.test.mjs new file mode 100644 index 0000000..1735df9 --- /dev/null +++ b/tests/agent-edge-api-routing.unit.test.mjs @@ -0,0 +1,220 @@ +/** + * Regression guard for the edge shadowing real API routes. + * + * `agent-edge.mjs` carried a `path.startsWith('/api/')` catch-all that answered + * 404 *before* OpenNext ever saw the request, with only `/api/ai` allow-listed + * above it. Every other `GET /api/*` — `/api/pages`, `/api/v1/agents`, the + * whole better-auth surface — was 404'd at the edge in production, while POST + * to the same paths reached its handler. + * + * These tests drive the real `worker.mjs` entrypoint with a stubbed OpenNext + * handler (see `fixtures/open-next-worker-stub.mjs`), so they cover the actual + * wiring, not a reimplementation of it. + * + * The route list is read off the filesystem rather than hard-coded: a route + * handler added under `src/app/api/` is covered the moment it lands, which is + * exactly the rot the original hand-maintained allow-list suffered. + */ +import { readdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import worker from '../worker.mjs'; +import { openNextStub } from './fixtures/open-next-worker-stub.mjs'; + +const API_DIR = resolve( + dirname(fileURLToPath(import.meta.url)), + '../src/app/api', +); + +/** Turn `src/app/api/pages/[pageId]/links/route.ts` into `/api/pages/sample/links`. */ +function collectApiRoutePaths(dir, prefix = '/api') { + const paths = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + const dynamic = entry.name.startsWith('['); + const catchAll = + entry.name.startsWith('[...') || entry.name.startsWith('[[...'); + const segment = dynamic + ? catchAll + ? 'sample/segment' + : 'sample' + : entry.name; + paths.push( + ...collectApiRoutePaths(join(dir, entry.name), `${prefix}/${segment}`), + ); + } else if (entry.name === 'route.ts' || entry.name === 'route.tsx') { + paths.push(prefix); + } + } + return paths; +} + +const API_ROUTE_PATHS = collectApiRoutePaths(API_DIR); + +const ctx = { + waitUntil: () => undefined, + passThroughOnException: () => undefined, +}; +const env = { NEXT_PUBLIC_APP_URL: 'https://karte.cc' }; + +function request(path, { method = 'GET', headers = {} } = {}) { + return worker.fetch( + new Request(`https://karte.cc${path}`, { + method, + headers: { host: 'karte.cc', ...headers }, + }), + env, + ctx, + ); +} + +beforeEach(() => { + openNextStub.reset(); +}); + +describe('edge does not shadow Next.js API routes', () => { + it('found the route handlers to guard', () => { + // Sanity check: if this ever hits zero the suite below is vacuous. + expect(API_ROUTE_PATHS.length).toBeGreaterThan(20); + expect(API_ROUTE_PATHS).toContain('/api/pages'); + expect(API_ROUTE_PATHS).toContain('/api/auth/sample/segment'); + }); + + it('reaches a real API route through the edge on GET', async () => { + openNextStub.handler = () => + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + + const response = await request('/api/pages'); + + expect(openNextStub.calls).toEqual(['/api/pages']); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + }); + + it.each(API_ROUTE_PATHS)('lets GET %s reach Next.js', async (path) => { + openNextStub.handler = () => new Response('routed', { status: 200 }); + + const response = await request(path); + + expect(openNextStub.calls).toEqual([path]); + expect(response.status).toBe(200); + }); + + it('lets HEAD reach Next.js too', async () => { + openNextStub.handler = () => new Response(null, { status: 200 }); + + const response = await request('/api/pages', { method: 'HEAD' }); + + expect(openNextStub.calls).toEqual(['/api/pages']); + expect(response.status).toBe(200); + }); + + it('does not let an Accept header divert an API route to the markdown 404', async () => { + openNextStub.handler = () => new Response('routed', { status: 200 }); + + const response = await request('/api/pages', { + headers: { accept: 'text/markdown' }, + }); + + expect(openNextStub.calls).toEqual(['/api/pages']); + expect(response.status).toBe(200); + }); +}); + +describe('unknown API paths still answer with the JSON 404 envelope', () => { + it('replaces the Next.js HTML 404 with JSON', async () => { + openNextStub.handler = () => + new Response('
404', { + status: 404, + headers: { 'content-type': 'text/html; charset=utf-8' }, + }); + + const response = await request('/api/definitely-not-a-real-path'); + + expect(openNextStub.calls).toEqual(['/api/definitely-not-a-real-path']); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('application/json'); + expect(await response.json()).toEqual({ + error: { + code: 'not_found', + message: 'Unknown API path: /api/definitely-not-a-real-path', + path: '/api/definitely-not-a-real-path', + }, + }); + }); + + it("leaves a route handler's own JSON 404 body untouched", async () => { + openNextStub.handler = () => + new Response(JSON.stringify({ error: 'page not found' }), { + status: 404, + headers: { 'content-type': 'application/json' }, + }); + + const response = await request('/api/pages/999999/links'); + + expect(await response.json()).toEqual({ error: 'page not found' }); + }); + + it('leaves non-API 404s alone so the HTML error page still renders', async () => { + openNextStub.handler = () => + new Response('404', { + status: 404, + headers: { 'content-type': 'text/html; charset=utf-8' }, + }); + + const response = await request('/no-such-page'); + + expect(openNextStub.calls).toEqual(['/no-such-page']); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('text/html'); + }); +}); + +describe('edge-owned surfaces still work', () => { + it('serves the agent catalog at /api/ai without touching Next.js', async () => { + const response = await request('/api/ai'); + + expect(openNextStub.calls).toEqual([]); + expect(response.status).toBe(200); + const catalog = await response.json(); + expect(catalog.name).toBe('Karte'); + expect(catalog.url).toBe('https://karte.cc'); + }); + + it('serves llms.txt, llms-full.txt, index.md, robots.txt and openapi.json', async () => { + for (const path of [ + '/llms.txt', + '/llms-full.txt', + '/index.md', + '/robots.txt', + '/openapi.json', + ]) { + expect((await request(path)).status, path).toBe(200); + } + expect(openNextStub.calls).toEqual([]); + }); + + it('still negotiates markdown on the homepage', async () => { + const response = await request('/', { + headers: { accept: 'text/markdown' }, + }); + + expect(openNextStub.calls).toEqual([]); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); + + it('still serves the markdown 404 for unknown non-API pages', async () => { + const response = await request('/no-such-page', { + headers: { accept: 'text/markdown' }, + }); + + expect(openNextStub.calls).toEqual([]); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); +}); diff --git a/tests/fixtures/cloudflare-workers-stub.mjs b/tests/fixtures/cloudflare-workers-stub.mjs new file mode 100644 index 0000000..b0152a0 --- /dev/null +++ b/tests/fixtures/cloudflare-workers-stub.mjs @@ -0,0 +1,11 @@ +/** + * Stand-in for the `cloudflare:workers` runtime module, which only exists + * inside workerd. `rate-limiter-do.mjs` imports `DurableObject` from it and is + * pulled in transitively by `worker.mjs`, the entrypoint under test. + */ +export class DurableObject { + constructor(ctx, env) { + this.ctx = ctx; + this.env = env; + } +} diff --git a/tests/fixtures/open-next-worker-stub.mjs b/tests/fixtures/open-next-worker-stub.mjs new file mode 100644 index 0000000..62fd40c --- /dev/null +++ b/tests/fixtures/open-next-worker-stub.mjs @@ -0,0 +1,37 @@ +/** + * Stand-in for `.open-next/worker.js` (a build artifact, gitignored, so it is + * absent in CI). `vitest.config.ts` aliases the OpenNext worker import to this + * module so `worker.mjs` — the real Cloudflare entrypoint — can be imported and + * exercised in a plain Node test. + * + * Tests set `openNextStub.handler` to decide what "Next.js" answers, and read + * `openNextStub.calls` to assert that a request actually reached it. + */ + +const unconfigured = () => + new Response('stub handler not configured', { status: 500 }); + +export const openNextStub = { + /** @type {(request: Request) => Response | Promise