From a7a385e6f1a110441585916be25b22aa238b651d Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Sat, 29 Aug 2026 22:51:10 +0530 Subject: [PATCH 1/3] fix(edge): stop agent-edge 404ing every real /api/* route on GET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `agent-edge.mjs` ran a subtractive catch-all before OpenNext: if (path.startsWith('/api/')) { return jsonError(404, 'not_found', `Unknown API path: ${path}`, path) } `handleAgentEdge` only engages for GET/HEAD, so every GET to a real `/api/*` route was 404'd at the edge and never reached its handler, while POST to the same path worked — the tell. Live against https://karte.cc: /api/welcome GET=404 POST=401 /api/pages GET=404 POST=401 ← exports GET /api/demo-chat GET=404 POST=400 /api/agent-waitlist GET=404 POST=400 /api/ai GET=200 ← edge-owned, allow-listed 21 of 58 route handlers under `src/app/api` export GET and were shadowed, including `/api/auth/[...all]` (better-auth), `/api/pages`, `/api/v1/agents`, `/api/settings/*` and a dozen `/api/pages/[pageId]/*`. The edge becomes a strict pre-handler: it answers only for the paths its additive `EXACT_ROUTES` allow-list grants it and returns `null` otherwise, so it can never conclude a path does not exist. The JSON-404 envelope moves to `withApiJsonNotFound`, a post-handler applied in `worker.mjs` to the response coming back from OpenNext — only Next.js's own 404 for an `/api/*` path becomes JSON, and a route handler's own JSON 404 is left alone. `/api/*` is also excluded from the markdown/Accept branch so an Accept header cannot divert a real API request. New routes now work with zero edge changes, which is why this shape cannot rot the way the hand-maintained allow-list did. `tests/agent-edge-api-routing.unit.test.mjs` drives the real `worker.mjs` entrypoint against a stubbed OpenNext handler and reads the route list off the filesystem, so newly added handlers are covered automatically. Reverting just the edge/worker changes fails 63 of its 69 assertions. `agent-edge.mjs` is a copied/generated fleet artifact, so `docs/architecture/edge-worker.md` carries a note that regenerating from an unfixed template reintroduces the outage. Co-Authored-By: Claude Opus 5 (1M context) --- agent-edge.mjs | 56 +++++- docs/architecture/edge-worker.md | 25 +++ docs/development/conventions.md | 3 + tests/agent-edge-api-routing.unit.test.mjs | 220 +++++++++++++++++++++ tests/fixtures/cloudflare-workers-stub.mjs | 11 ++ tests/fixtures/open-next-worker-stub.mjs | 37 ++++ vitest.config.ts | 23 ++- worker.mjs | 19 +- 8 files changed, 383 insertions(+), 11 deletions(-) create mode 100644 tests/agent-edge-api-routing.unit.test.mjs create mode 100644 tests/fixtures/cloudflare-workers-stub.mjs create mode 100644 tests/fixtures/open-next-worker-stub.mjs diff --git a/agent-edge.mjs b/agent-edge.mjs index 365794a8..4ed4b2e4 100644 --- a/agent-edge.mjs +++ b/agent-edge.mjs @@ -2,10 +2,16 @@ * Portable agent-edge handler — copy or generate into each product. * Spec: fleet-ops/docs/agent-indexing-standard.md * - * Usage in worker.mjs (before openNext.fetch): - * import { handleAgentEdge } from './agent-edge.mjs' + * The edge is a *strict pre-handler*: it answers only for the surfaces it + * genuinely owns and returns `null` for everything else. It must never decide + * that a path does not exist — only Next.js knows the route table. + * + * Usage in worker.mjs: + * import { handleAgentEdge, withApiJsonNotFound } from './agent-edge.mjs' * const agent = handleAgentEdge(request) * if (agent) return agent + * // ...and on the way back out, so unknown /api/* paths answer in JSON: + * return withApiJsonNotFound(request, await openNext.fetch(request, env, ctx)) */ import { @@ -574,6 +580,15 @@ function catalogFor(origin) { } /** + * Pre-handler: answers only for surfaces the edge itself owns, and returns + * `null` for everything else so the request reaches OpenNext/Next.js. + * + * The allow-list (`EXACT_ROUTES`) is additive — it *grants* the edge specific + * paths. There is deliberately no subtractive `/api/*` catch-all here: the edge + * does not know the Next.js route table and must never answer 404 on its + * behalf. Unknown `/api/*` paths are shaped into JSON by `withApiJsonNotFound` + * on the way back out instead. + * * @param {Request} request * @returns {Response | null} */ @@ -586,10 +601,10 @@ export function handleAgentEdge(request) { const exactResponse = exact ? exact(url) : null; if (exactResponse) return exactResponse; - // JSON errors for unknown /api/* paths. - if (path.startsWith('/api/')) { - return jsonError(404, 'not_found', `Unknown API path: ${path}`, path); - } + // `/api/*` belongs to Next.js route handlers. Fall through unconditionally — + // an Accept header must never divert a real API request away from its + // handler, and the edge must never claim the path is missing. + if (isApiPath(path)) return null; if (!wantsMarkdown(request)) return null; @@ -607,6 +622,35 @@ export function handleAgentEdge(request) { return null; } +/** + * Post-handler: shape Next.js's own 404 for `/api/*` into a JSON error body. + * + * The edge deliberately does not know which API routes exist — Next.js does. + * We call it, and only if *it* reports 404 do we swap the HTML error page for + * the machine-readable JSON envelope. That is why this shape cannot rot the way + * the previous `/api/*` catch-all did: a route handler added under + * `src/app/api/` is reachable with no edge change, because the edge never + * asserts non-existence. + * + * @param {Request} request + * @param {Response} response Response from the downstream Next.js handler. + * @returns {Response} + */ +export function withApiJsonNotFound(request, response) { + if (request.method !== 'GET' && request.method !== 'HEAD') return response; + if (response.status !== 404) return response; + const path = new URL(request.url).pathname; + if (!isApiPath(path)) return response; + const contentType = response.headers.get('content-type') || ''; + // A route handler's own JSON 404 is already machine-readable — leave it. + if (contentType.includes('application/json')) return response; + return jsonError(404, 'not_found', `Unknown API path: ${path}`, path); +} + +function isApiPath(pathname) { + return pathname.startsWith('/api/'); +} + /** * Exact-path agent surfaces. Each handler takes the request URL so it can * bind responses to the origin that was asked for. diff --git a/docs/architecture/edge-worker.md b/docs/architecture/edge-worker.md index c451eb27..ce622252 100644 --- a/docs/architecture/edge-worker.md +++ b/docs/architecture/edge-worker.md @@ -28,6 +28,31 @@ live in `worker.mjs` / `worker-routing.mjs` / `agent-edge.mjs` instead. | `rate-limiter-do.mjs` | `RateLimiterDO` Durable Object backing `src/lib/rate-limit.ts`. | | `timing.mjs` | `withTiming()` wrapper for request timing. | +## The edge never owns `/api/*` it did not declare + +`agent-edge.mjs` is a **strict pre-handler**. It answers only for the exact +paths in its `EXACT_ROUTES` allow-list (`/llms.txt`, `/llms-full.txt`, +`/index.md`, `/robots.txt`, `/openapi.json`, `/openapi.yaml`, `/api/ai`) and +returns `null` for everything else, so the request reaches OpenNext. It must +never conclude that a path does *not* exist — only Next.js knows the route +table. + +The JSON error envelope for unknown API paths is a **post-handler**, +`withApiJsonNotFound(request, response)`, applied in `worker.mjs` to the +response coming back from OpenNext: only when Next.js itself answers 404 for an +`/api/*` path does the HTML error page become +`{"error":{"code":"not_found",…}}`. A route handler's own JSON 404 is left +untouched. + +> **Carry-forward when regenerating `agent-edge.mjs`.** This file is a +> copied/generated fleet artifact (see its header and the +> `apply-agent-surfaces` payload marker). The upstream template's `/api/*` +> catch-all 404 shadowed every real Next.js route handler in production — +> `GET /api/pages` returned the edge's `not_found` envelope while `POST` to the +> same path reached its handler. Regenerating from an unfixed template +> reintroduces the outage. `tests/agent-edge-api-routing.unit.test.mjs` fails +> loudly if it comes back; do not "fix" it by narrowing the test. + ## Cacheable document paths `worker.mjs` keeps a `CACHEABLE_EXACT` set of landing/marketing document paths diff --git a/docs/development/conventions.md b/docs/development/conventions.md index 5823822f..29cf5731 100644 --- a/docs/development/conventions.md +++ b/docs/development/conventions.md @@ -31,6 +31,9 @@ React Compiler is **ON** (`babel-plugin-react-compiler`). - `middleware.ts` / `proxy.ts` for edge guards — not supported by the Cloudflare OpenNext adapter. Use `worker.mjs` / `worker-routing.mjs` / `agent-edge.mjs` (ADR 0001). +- A subtractive `/api/*` catch-all in `agent-edge.mjs`. It 404'd every real + Next.js route handler on GET in production. See the carry-forward note in + `docs/architecture/edge-worker.md`. - SaaS Maker RAG as a profile-memory fallback — removed; only the shared `knowledgebase` Worker is used (`docs/architecture/rag-memory.md`). - The legacy `unsafe` native ratelimit binding — replaced by `RateLimiterDO` diff --git a/tests/agent-edge-api-routing.unit.test.mjs b/tests/agent-edge-api-routing.unit.test.mjs new file mode 100644 index 00000000..1735df9a --- /dev/null +++ b/tests/agent-edge-api-routing.unit.test.mjs @@ -0,0 +1,220 @@ +/** + * Regression guard for the edge shadowing real API routes. + * + * `agent-edge.mjs` carried a `path.startsWith('/api/')` catch-all that answered + * 404 *before* OpenNext ever saw the request, with only `/api/ai` allow-listed + * above it. Every other `GET /api/*` — `/api/pages`, `/api/v1/agents`, the + * whole better-auth surface — was 404'd at the edge in production, while POST + * to the same paths reached its handler. + * + * These tests drive the real `worker.mjs` entrypoint with a stubbed OpenNext + * handler (see `fixtures/open-next-worker-stub.mjs`), so they cover the actual + * wiring, not a reimplementation of it. + * + * The route list is read off the filesystem rather than hard-coded: a route + * handler added under `src/app/api/` is covered the moment it lands, which is + * exactly the rot the original hand-maintained allow-list suffered. + */ +import { readdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import worker from '../worker.mjs'; +import { openNextStub } from './fixtures/open-next-worker-stub.mjs'; + +const API_DIR = resolve( + dirname(fileURLToPath(import.meta.url)), + '../src/app/api', +); + +/** Turn `src/app/api/pages/[pageId]/links/route.ts` into `/api/pages/sample/links`. */ +function collectApiRoutePaths(dir, prefix = '/api') { + const paths = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + const dynamic = entry.name.startsWith('['); + const catchAll = + entry.name.startsWith('[...') || entry.name.startsWith('[[...'); + const segment = dynamic + ? catchAll + ? 'sample/segment' + : 'sample' + : entry.name; + paths.push( + ...collectApiRoutePaths(join(dir, entry.name), `${prefix}/${segment}`), + ); + } else if (entry.name === 'route.ts' || entry.name === 'route.tsx') { + paths.push(prefix); + } + } + return paths; +} + +const API_ROUTE_PATHS = collectApiRoutePaths(API_DIR); + +const ctx = { + waitUntil: () => undefined, + passThroughOnException: () => undefined, +}; +const env = { NEXT_PUBLIC_APP_URL: 'https://karte.cc' }; + +function request(path, { method = 'GET', headers = {} } = {}) { + return worker.fetch( + new Request(`https://karte.cc${path}`, { + method, + headers: { host: 'karte.cc', ...headers }, + }), + env, + ctx, + ); +} + +beforeEach(() => { + openNextStub.reset(); +}); + +describe('edge does not shadow Next.js API routes', () => { + it('found the route handlers to guard', () => { + // Sanity check: if this ever hits zero the suite below is vacuous. + expect(API_ROUTE_PATHS.length).toBeGreaterThan(20); + expect(API_ROUTE_PATHS).toContain('/api/pages'); + expect(API_ROUTE_PATHS).toContain('/api/auth/sample/segment'); + }); + + it('reaches a real API route through the edge on GET', async () => { + openNextStub.handler = () => + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + + const response = await request('/api/pages'); + + expect(openNextStub.calls).toEqual(['/api/pages']); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + }); + + it.each(API_ROUTE_PATHS)('lets GET %s reach Next.js', async (path) => { + openNextStub.handler = () => new Response('routed', { status: 200 }); + + const response = await request(path); + + expect(openNextStub.calls).toEqual([path]); + expect(response.status).toBe(200); + }); + + it('lets HEAD reach Next.js too', async () => { + openNextStub.handler = () => new Response(null, { status: 200 }); + + const response = await request('/api/pages', { method: 'HEAD' }); + + expect(openNextStub.calls).toEqual(['/api/pages']); + expect(response.status).toBe(200); + }); + + it('does not let an Accept header divert an API route to the markdown 404', async () => { + openNextStub.handler = () => new Response('routed', { status: 200 }); + + const response = await request('/api/pages', { + headers: { accept: 'text/markdown' }, + }); + + expect(openNextStub.calls).toEqual(['/api/pages']); + expect(response.status).toBe(200); + }); +}); + +describe('unknown API paths still answer with the JSON 404 envelope', () => { + it('replaces the Next.js HTML 404 with JSON', async () => { + openNextStub.handler = () => + new Response('404', { + status: 404, + headers: { 'content-type': 'text/html; charset=utf-8' }, + }); + + const response = await request('/api/definitely-not-a-real-path'); + + expect(openNextStub.calls).toEqual(['/api/definitely-not-a-real-path']); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('application/json'); + expect(await response.json()).toEqual({ + error: { + code: 'not_found', + message: 'Unknown API path: /api/definitely-not-a-real-path', + path: '/api/definitely-not-a-real-path', + }, + }); + }); + + it("leaves a route handler's own JSON 404 body untouched", async () => { + openNextStub.handler = () => + new Response(JSON.stringify({ error: 'page not found' }), { + status: 404, + headers: { 'content-type': 'application/json' }, + }); + + const response = await request('/api/pages/999999/links'); + + expect(await response.json()).toEqual({ error: 'page not found' }); + }); + + it('leaves non-API 404s alone so the HTML error page still renders', async () => { + openNextStub.handler = () => + new Response('404', { + status: 404, + headers: { 'content-type': 'text/html; charset=utf-8' }, + }); + + const response = await request('/no-such-page'); + + expect(openNextStub.calls).toEqual(['/no-such-page']); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('text/html'); + }); +}); + +describe('edge-owned surfaces still work', () => { + it('serves the agent catalog at /api/ai without touching Next.js', async () => { + const response = await request('/api/ai'); + + expect(openNextStub.calls).toEqual([]); + expect(response.status).toBe(200); + const catalog = await response.json(); + expect(catalog.name).toBe('Karte'); + expect(catalog.url).toBe('https://karte.cc'); + }); + + it('serves llms.txt, llms-full.txt, index.md, robots.txt and openapi.json', async () => { + for (const path of [ + '/llms.txt', + '/llms-full.txt', + '/index.md', + '/robots.txt', + '/openapi.json', + ]) { + expect((await request(path)).status, path).toBe(200); + } + expect(openNextStub.calls).toEqual([]); + }); + + it('still negotiates markdown on the homepage', async () => { + const response = await request('/', { + headers: { accept: 'text/markdown' }, + }); + + expect(openNextStub.calls).toEqual([]); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); + + it('still serves the markdown 404 for unknown non-API pages', async () => { + const response = await request('/no-such-page', { + headers: { accept: 'text/markdown' }, + }); + + expect(openNextStub.calls).toEqual([]); + expect(response.status).toBe(404); + expect(response.headers.get('content-type')).toContain('text/markdown'); + }); +}); diff --git a/tests/fixtures/cloudflare-workers-stub.mjs b/tests/fixtures/cloudflare-workers-stub.mjs new file mode 100644 index 00000000..b0152a0f --- /dev/null +++ b/tests/fixtures/cloudflare-workers-stub.mjs @@ -0,0 +1,11 @@ +/** + * Stand-in for the `cloudflare:workers` runtime module, which only exists + * inside workerd. `rate-limiter-do.mjs` imports `DurableObject` from it and is + * pulled in transitively by `worker.mjs`, the entrypoint under test. + */ +export class DurableObject { + constructor(ctx, env) { + this.ctx = ctx; + this.env = env; + } +} diff --git a/tests/fixtures/open-next-worker-stub.mjs b/tests/fixtures/open-next-worker-stub.mjs new file mode 100644 index 00000000..62fd40c2 --- /dev/null +++ b/tests/fixtures/open-next-worker-stub.mjs @@ -0,0 +1,37 @@ +/** + * Stand-in for `.open-next/worker.js` (a build artifact, gitignored, so it is + * absent in CI). `vitest.config.ts` aliases the OpenNext worker import to this + * module so `worker.mjs` — the real Cloudflare entrypoint — can be imported and + * exercised in a plain Node test. + * + * Tests set `openNextStub.handler` to decide what "Next.js" answers, and read + * `openNextStub.calls` to assert that a request actually reached it. + */ + +const unconfigured = () => + new Response('stub handler not configured', { status: 500 }); + +export const openNextStub = { + /** @type {(request: Request) => Response | Promise} */ + handler: unconfigured, + /** @type {string[]} */ + calls: [], + reset() { + openNextStub.calls = []; + openNextStub.handler = unconfigured; + }, +}; + +const worker = { + async fetch(request) { + openNextStub.calls.push(new URL(request.url).pathname); + return await openNextStub.handler(request); + }, +}; + +export default worker; + +// `worker.mjs` re-exports these Durable Object classes from the OpenNext entry. +export class BucketCachePurge {} +export class DOQueueHandler {} +export class DOShardedTagCache {} diff --git a/vitest.config.ts b/vitest.config.ts index 661ef79a..0821653b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,7 +3,28 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ resolve: { - alias: { '@': resolve(__dirname, 'src') }, + alias: [ + // `.open-next/worker.js` is a gitignored build artifact, so it does not + // exist in CI. Alias it to a stub so `worker.mjs` — the real Cloudflare + // entrypoint — can be imported and exercised by unit tests. + { + find: /^\.\/\.open-next\/worker\.js$/, + replacement: resolve( + __dirname, + 'tests/fixtures/open-next-worker-stub.mjs', + ), + }, + // `cloudflare:workers` only exists inside workerd; `rate-limiter-do.mjs` + // (imported by `worker.mjs`) needs `DurableObject` from it. + { + find: 'cloudflare:workers', + replacement: resolve( + __dirname, + 'tests/fixtures/cloudflare-workers-stub.mjs', + ), + }, + { find: '@', replacement: resolve(__dirname, 'src') }, + ], }, test: { environment: 'node', diff --git a/worker.mjs b/worker.mjs index 03ecd063..51aa00da 100644 --- a/worker.mjs +++ b/worker.mjs @@ -16,7 +16,7 @@ import openNext, { DOQueueHandler as OpenNextDOQueueHandler, DOShardedTagCache as OpenNextDOShardedTagCache, } from './.open-next/worker.js'; -import { handleAgentEdge } from './agent-edge.mjs'; +import { handleAgentEdge, withApiJsonNotFound } from './agent-edge.mjs'; import { handlePublicRouteMarkdown } from './public-route-markdown.mjs'; import { RateLimiterDO as RateLimiterDurableObject } from './rate-limiter-do.mjs'; import { withTiming } from './timing.mjs'; @@ -60,7 +60,8 @@ function isCacheableDocumentPath(pathname) { } export default { fetch: withTiming(async function fetch(request, env, ctx) { - // Agent / LLM indexing surfaces (fleet GEO standard) + // Agent / LLM indexing surfaces (fleet GEO standard). This only answers + // for paths the edge itself owns; everything else falls through below. { const agent = handleAgentEdge(request); if (agent) return agent; @@ -93,12 +94,22 @@ export default { ); if (markdown) return markdown; + // `withApiJsonNotFound` is applied on the way back out: Next.js owns the + // route table, so only its 404 (never an edge guess) turns into a JSON + // error body for `/api/*`. These two branches are the only ones an + // `/api/*` request can reach — API paths are never cacheable documents. if (request.method !== 'GET') { - return openNext.fetch(request, env, ctx); + return withApiJsonNotFound( + request, + await openNext.fetch(request, env, ctx), + ); } const url = new URL(request.url); if (!isCacheableDocumentPath(url.pathname)) { - const response = await openNext.fetch(request, env, ctx); + const response = withApiJsonNotFound( + request, + await openNext.fetch(request, env, ctx), + ); return routed.cacheProfile ? addProfileCacheHeaders(response) : response; From 8724626826e2598bb2cec89567b963fc9ba5f94d Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Sat, 29 Aug 2026 22:54:08 +0530 Subject: [PATCH 2/3] chore: format scorecard.json to unblock CI `pnpm quality` starts with `biome check .` and was aborting on .fleet/evidence/landing-audit/scorecard.json, which was committed unformatted in 00683c0. Because that step aborts before the test step ever runs, CI is red on main and no PR in this repo can go green. This is unrelated to the edge-routing fix in this PR and is a pure formatting change (biome check --write, no content edited). It is included here only because it blocks this PR from merging. Co-Authored-By: Claude Opus 5 (1M context) --- .fleet/evidence/landing-audit/scorecard.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.fleet/evidence/landing-audit/scorecard.json b/.fleet/evidence/landing-audit/scorecard.json index e3a78696..d97fd553 100644 --- a/.fleet/evidence/landing-audit/scorecard.json +++ b/.fleet/evidence/landing-audit/scorecard.json @@ -41,7 +41,13 @@ }, "geo": { "productionAgentIndex": "S 100%", - "surfaces": ["/llms.txt", "/llms-full.txt", "/index.md", "/api/ai", "/skill.md"] + "surfaces": [ + "/llms.txt", + "/llms-full.txt", + "/index.md", + "/api/ai", + "/skill.md" + ] }, "performance": { "tool": "PSI Swarm 0.4.2", From 01abcc4207833b6bb42aa1e4f1a1cfe2b9b4a581 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Sat, 29 Aug 2026 22:57:30 +0530 Subject: [PATCH 3/3] chore: exclude test fixtures from the knip scan The new worker/OpenNext stubs under tests/fixtures are wired in through vitest.config.ts aliases rather than imported by path, so knip cannot see the reference and reported them as 1 unused file and 4 unused exports, failing `pnpm quality:unused`. knip.json already ignores tests/e2e/**; this extends the same treatment to tests/fixtures/**. Verified against main: main reports 0 unused files and 0 unused exports, so this restores that baseline rather than widening it to hide anything real. Co-Authored-By: Claude Opus 5 (1M context) --- knip.json | 1 + 1 file changed, 1 insertion(+) diff --git a/knip.json b/knip.json index bf7fc80b..216629ec 100644 --- a/knip.json +++ b/knip.json @@ -22,6 +22,7 @@ "**/eslint.config.*", "scripts/**", "tests/e2e/**", + "tests/fixtures/**", "test/**", "**/*.test.{ts,tsx,js,mjs}", "**/*.spec.{ts,tsx,js,mjs}",