diff --git a/packages/plugin/METRICS.md b/packages/plugin/METRICS.md index fb84624..613b850 100644 --- a/packages/plugin/METRICS.md +++ b/packages/plugin/METRICS.md @@ -121,17 +121,17 @@ PK drives the scan (an open range can make the planner walk a metric's entire hi One-line summaries; `src/metrics.js` carries the full description of every dimension value and the reasoning behind it. -| Metric | Kind | `path` | `method` | `type` | What it's for | -| ---------------- | ------- | ---------- | ----------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | -| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | -| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | -| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | -| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | -| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — the render-failure alert). | -| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | -| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*`, `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope). | -| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `reconcile_restored`/`reconcile_missing` (per sweep). | +| Metric | Kind | `path` | `method` | `type` | What it's for | +| ---------------- | ------- | ---------- | ----------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `bot_request` | counter | host | botName | deviceType | Raw crawl volume and mix at ingress. The denominator for every serve-side ratio. | +| `bot_serve` | counter | source | cacheStatus | botName | **Origin offload** and **cache hit rate** — the two rollout numbers. | +| `route_serve` | counter | route | cacheStatus | deviceType | The same outcome per route: which route's `renderInterval` needs to move. | +| `page_age` | ms | botName | deviceType | — | Freshness as delivered: ms since the served snapshot rendered (cache serves only). | +| `route_page_age` | ms | route | cacheStatus | deviceType | Served age per route, split by freshness state — the "should this TTL move" number. | +| `render` | value | series | per-series | per-series | The render fleet in one scan: `time_ms` (duration by statusCode × candidacy — renders/hour = concurrency ÷ time_ms) and `outcome` (counter by outcome × detail, exactly one per posted result — the render-failure alert). | +| `origin_fetch` | ms | statusCode | reason | — | Cost of every non-cache serve: origin latency + status, by why the cache didn't answer (miss/stale/skip/invalidated/bypass/blob-missing/blob-timeout/render-timeout). | +| `prerender_ops` | value | series | detail | context | Every low-volume ops signal in one scan: `unrouted` (class, bucket), `sitemap_*`, `serve_error`, `config_warnings`, `page_age_negative` (bot, device), `demand_*` (ladder decisions + `fast_fraction`/`fill`), `invalidation_error` (kind), `invalidation_reenqueue` (outcome, scope). | +| `queue_health` | value | series | result | — | Every queue signal in one scan: the snapshot gauges (`overdue`, `lease_occupancy`, `below_floor`, `below_floor_age_ms`, `floor_pin_age_ms`, `paused`), `claim_scan_ms` (per pass, method = granted/empty/capped), `lane_granted` (per lane per pass, method = the lane label), `reconcile_restored`/`reconcile_missing` (per sweep). | Notes that bite: @@ -358,19 +358,20 @@ The catalog above is reference; this is the short list. "Sum across nodes" is im **Thresholds — warn, then investigate:** -| Condition | Meaning | -| ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -| `queue_health` `overdue` − `lease_occupancy` growing snapshot-over-snapshot | The fleet is falling behind demand (remember: `overdue`'s healthy floor IS the in-flight count). | -| `queue_health` `floor_pin_age_ms` > ~1 h | One key is holding the claim scan's seek position — the whole node's queue ages behind it. | -| `bot_serve` swr share rising / `route_page_age` p95 > that route's `renderInterval` | The cadence is configured but not delivered — a capacity or scheduling problem, not a config one. | -| `bot_serve` miss share rising | Coverage: new URLs the corpus doesn't have, or the CDN forwarding paths it shouldn't (check `unrouted`). | -| `duration` p95 (`path: 'p'`) or `success` ratio degrading | The crawler-facing SLO, independent of any plugin-level explanation. | -| `queue_status` report timestamp stale, or intent ≠ observed > one sync interval | A node stopped reporting (and likely claiming), or pause propagation is stuck. | -| `render` outcome `suppressed` or `failed` share rising | Mass suppression (an origin change disavowing pages) or a failing fleet — shares are readable directly because outcomes sum to results. | -| `queue_health` `claim_scan_ms` p95 trending up | The scan is degrading (dead index entries at the seek point) before any backlog shows. Watch the trend, not the absolute number. | -| `origin_fetch` p95 or 5xx/`0` share rising | Origin trouble that bots feel directly on every miss; a rising `render-timeout` share is renderNow falling back. | -| `queue_health` `paused` = 1 beyond the expected window | A node's queue is paused longer than whoever paused it intended. | -| `prerender_ops` series `config_warnings` changed after a deploy | The deploy introduced a finding; `GET /prerender_admin/config` names it. | +| Condition | Meaning | +| ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `queue_health` `overdue` − `lease_occupancy` growing snapshot-over-snapshot | The fleet is falling behind demand (remember: `overdue`'s healthy floor IS the in-flight count). | +| `queue_health` `floor_pin_age_ms` > ~1 h | One key is holding the claim scan's seek position — the whole node's queue ages behind it. | +| `bot_serve` swr share rising / `route_page_age` p95 > that route's `renderInterval` | The cadence is configured but not delivered — a capacity or scheduling problem, not a config one. | +| `bot_serve` miss share rising | Coverage: new URLs the corpus doesn't have, or the CDN forwarding paths it shouldn't (check `unrouted`). | +| `duration` p95 (`path: 'p'`) or `success` ratio degrading | The crawler-facing SLO, independent of any plugin-level explanation. | +| `queue_status` report timestamp stale, or intent ≠ observed > one sync interval | A node stopped reporting (and likely claiming), or pause propagation is stuck. | +| `render` outcome `suppressed` or `failed` share rising | Mass suppression (an origin change disavowing pages) or a failing fleet — shares are readable directly because outcomes sum to results. | +| `queue_health` `claim_scan_ms` p95 trending up | The scan is degrading (dead index entries at the seek point) before any backlog shows. Watch the trend, not the absolute number. | +| `queue_health` `lane_granted` at zero for a lane whose backlog is not empty | That lane is being starved. Either its `queue.lanes.minShare` floor is too low, or `ttlBands` put two very different cadences in one band so a slower route is consuming it. Lanes move no totals, so nothing else shows this. | +| `origin_fetch` p95 or 5xx/`0` share rising | Origin trouble that bots feel directly on every miss; a rising `render-timeout` share is renderNow falling back. | +| `queue_health` `paused` = 1 beyond the expected window | A node's queue is paused longer than whoever paused it intended. | +| `prerender_ops` series `config_warnings` changed after a deploy | The deploy introduced a finding; `GET /prerender_admin/config` names it. | **Absence is a signal — alert when a series stops:** diff --git a/packages/plugin/README.md b/packages/plugin/README.md index 4c1695a..5389c7d 100644 --- a/packages/plugin/README.md +++ b/packages/plugin/README.md @@ -518,6 +518,95 @@ The floor advances to **the first due row a pass observed**, which is the same t Set `queue.claimFloor.enabled: false` to roll the floor back to the old full seek. It changes nothing else; leases stay where they are either way. +#### Render lanes: which due row goes first + +The floor decides _where the scan starts_. `queue.lanes` decides _which scan_ — and that distinction +is the whole feature, because a single scan can only start in one place. + +`claim` is strictly `nextRenderTime`-ascending, so under any capacity deficit the queue serves +whatever is oldest-due. Two production measurements say that is the wrong order +([#80](https://github.com/HarperFast/prerender-plugin/issues/80)): + +- **Provenance.** During a multi-hour backlog, **239,090 of 521,929 overdue rows (~46%)** were + bot-discovered rather than sitemap-submitted — half the capacity going to pages nobody submitted + while submitted pages aged out of their SWR window. +- **TTL-blindness**, which is sharper. Absolute due time treats a 1 h-TTL homepage 3 h overdue exactly + like a 48 h-TTL product page 3 h overdue. The first is 300% stale, the second 6%. Simulated over the + real corpus, the 1 h route sits at **4.78× its own TTL even at full capacity** and 48.83× at half, + against 1.08× / 2.00× for the 48 h route. + +**Why not just rank the due window by relative lateness.** Because it cannot be an _order_: +`(t − dueAt) / interval` is linear in `t` with slope `1/interval`, so two rows with different +intervals cross exactly once, and no stored key can express an order that changes with the clock. +Re-ranking inside the claim window does not rescue it either — the window is anchored at the _oldest_ +due time and is already an EDF prefix, so a homepage 600 minutes down the queue is never read at all. +Widening the window does not help for the same reason. + +So relative lateness is the _rationale_ for which lane a row is in, never the comparator. Lanes are +cut so intervals inside one are similar; within a lane the order is unchanged (earliest due first), +which is optimal for maximum lateness. + +**The encoding is `nextRenderTime = lane × 2^42 + dueAt`.** One column, the existing index, no schema +change, no second index, no migration — every pre-existing row is already a valid lane-0 row, and a +lane change is an in-place numeric update. Measured: three lanes interleaved in one index, each with +its own watermark, read in **0.29–0.32 ms per lane**; the same lane with its watermark reset costs +**3.46 ms**. Interleaving is free — the watermark is the entire win, which is why each lane gets one +(in its own shared buffer, so `renderLease.js` keeps its layout and its invariants). + +Lanes, in priority order, all **derived at write time** from provenance, cadence and failure count — +so a band or route change applies on each URL's next render with no sweep: + +| lane | from | +| ------------------ | ------------------------------------------------------------------------------ | +| `urgent` | operator intent — the admin re-render action, and an authenticated `renderNow` | +| `submitted/b0…bN` | has a `sitemapUrl`, banded by `queue.lanes.ttlBands` | +| `discovered/b0…bN` | crawler-found, banded the same way | +| `cold` | past the fast-retry lane, or a suppressed target's recheck | + +**Fairness is scheduler policy, deliberately not part of the key** — so it is tunable live with no +rewriting of stored rows. `urgentMaxShare` caps lane 0's _drain share_ rather than its admission: a +hard structural bound that needs no token bucket, so a bulk force-render cannot take more than that +fraction of any batch. `minShare` reserves **floors**, not fixed shares: strict priority starves the +tail at _every_ capacity level including 100% (and its lag numbers look good precisely because it +drops work — starvation is invisible in a lag metric), while fixed shares summing to 1.0 leave nothing +for the priority order to spend. Floors measured far better in the tail — discovery reaching 71 h +instead of 133 h at 50% capacity. An unclaimed floor is released back to lane order within the same +pass, so a floor for a class with nothing due costs nothing. + +**Enabling it is two steps, and the order matters.** Lane 0 is `urgent` _and_ lane 0 is the unencoded +value — that is what makes the encoding migration-free, and it is also the trap: on a fresh deploy the +whole corpus reads as urgent, and `urgentMaxShare` would ration the queue to a fifth of capacity. So: + +```sh +# 1. re-stamp: rewrites each due time into its derived lane, in place, changing NO due time. +# Bounded and repeatable — call until "done": true. +curl -sk -X POST https://:9926/prerender_admin/queue -u -H 'content-type: application/json' -d '{"action":"restamp-lanes","limit":5000}' +``` + +```yaml +# 2. then, in config.yaml: +queue: + lanes: + enabled: true +``` + +The re-stamp refuses to run once `enabled` is true, because at that point a lane-0 row can no longer +be told apart from one an operator deliberately marked urgent. It cannot derive a stored `changefreq` +interval, a demand-ladder rung or `cold` (those live on the Target, and reading it would be a point +read per row on an 814k corpus), so a promoted catalog page is stamped one band slow and a suppressed +URL one lane fast — both corrected on that row's next render. Being one band optimistic for one cycle +is the right direction to be wrong in. + +**Disabling is not an instant rollback.** Rows already encoded stay encoded until each renders again, +and while lanes are off they sort after every unencoded row, so they drain slowly rather than +promptly. Nothing is stranded — reads decode unconditionally — but to get the old behaviour back at +once, re-stamp with lanes disabled, which writes every row back to lane 0. + +Watch `queue_health` `lane_granted` (jobs granted per pass, lane in the method slot). Lanes +redistribute a fixed amount of work, so no total moves when they engage and this is the only series +that shows the split — a lane sitting at zero while its backlog is non-empty is the failure to look +for, and it is indistinguishable from health anywhere else. + ## HTTP & resource API | Method & path | Purpose | diff --git a/packages/plugin/package.json b/packages/plugin/package.json index d84ab29..77ef186 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.49.0", + "version": "0.50.0", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/configSchema.js b/packages/plugin/src/configSchema.js index 479a6bd..5726d23 100644 --- a/packages/plugin/src/configSchema.js +++ b/packages/plugin/src/configSchema.js @@ -1262,6 +1262,94 @@ export const configSchema = group('Prerender plugin configuration.', { 'about could not be detected.', { min: 1, scope: 'restart' } ), + lanes: group( + 'RENDER PRIORITY. `claim` is strictly `nextRenderTime`-ascending, so under any capacity ' + + 'deficit the queue serves whatever is oldest-due — regardless of whether the site owner ' + + 'submitted the page or a crawler found it, and regardless of how tight its freshness ' + + 'budget is. Two production measurements say that is the wrong order (see ' + + 'prerender-plugin#80):\n\n' + + ' PROVENANCE: during a multi-hour backlog, 239,090 of 521,929 overdue rows (~46%) were ' + + 'bot-discovered rather than sitemap-submitted — half the render capacity spent on pages ' + + 'nobody submitted, while submitted pages aged out of their SWR window.\n\n' + + ' TTL-BLINDNESS: absolute due time treats a 1h-TTL homepage 3h overdue exactly like a ' + + '48h-TTL product page 3h overdue. The first is 300% stale, the second 6%. Simulated over ' + + 'the real corpus, the 1h route sits at 4.78x its own TTL even at FULL capacity and ' + + '48.83x at half, against 1.08x / 2.00x for the 48h route.\n\n' + + 'Lanes fix the ORDER, never the volume: the same number of pages render either way. A ' + + 'lane is encoded in the high bits of `nextRenderTime` (`lane x 2^42 + dueAt`), so there ' + + 'is no schema change, no second index and no migration — every pre-existing row is ' + + 'already a valid lane-0 row. Within a lane the order is unchanged (earliest due first), ' + + 'which is optimal for maximum lateness; lanes are cut so the intervals inside one are ' + + 'similar, and the lane a row belongs to is DERIVED at write time from its provenance, ' + + 'its cadence and its failure count, so a config change here applies on each URL’s next ' + + 'render with no sweep.\n\n' + + 'READ `enabled` BEFORE TURNING THIS ON. The rollout is two steps and the order matters.', + { + enabled: option( + false, + 'Off by default, and enabling it is a TWO-STEP OPERATION.\n\n' + + 'Lane 0 is `urgent` and lane 0 is the UNENCODED value — which is what makes the encoding ' + + 'migration-free, and is also the trap: every row written before lanes existed is ' + + 'numerically in lane 0, so on a fresh deploy the WHOLE CORPUS reads as urgent and ' + + '`urgentMaxShare` would ration the entire queue down to a fifth of capacity.\n\n' + + 'So: (1) `POST /prerender_admin/queue {"action":"restamp-lanes"}` and let it finish — it ' + + 'walks the schedule rows and rewrites each due time into its derived lane, in place, ' + + 'changing no due time; (2) set this true. Both steps are safe in either order in the ' + + 'sense that nothing is lost, but doing (2) first means running with the whole corpus in ' + + 'lane 0 until (1) completes.\n\n' + + 'DISABLING IS NOT AN INSTANT ROLLBACK. Rows already encoded stay encoded until each one ' + + 'renders again, and while this is false they sort after every unencoded row — so they ' + + 'drain slowly rather than promptly. Nothing is stranded (reads decode unconditionally), ' + + 'but if you need the old behaviour back at once, re-stamp with lanes disabled, which ' + + 'writes every row back to lane 0.' + ), + ttlBands: option( + [HOUR, 12 * HOUR], + 'Cadence cuts (ms, ascending) that split `submitted` and `discovered` into lanes. A row ' + + 'goes in the first band its render interval does not exceed; anything slower lands in the ' + + 'overflow band, so N cuts make N+1 bands.\n\n' + + 'This is the half that fixes TTL-blindness, and it is why the coarse classes alone are ' + + 'not enough: a class whose intervals are NOT similar reproduces the problem inside itself. ' + + 'At the default, a 1h homepage, a 6-12h catalog page and a 48h product page occupy three ' + + 'different lanes, so the homepage stops queueing behind product pages that are older in ' + + 'absolute terms but barely stale in their own terms.\n\n' + + 'Set cuts at your ACTUAL route cadences, not at round numbers — a band that contains two ' + + 'route cadences an order of magnitude apart is a band that does nothing. An empty list ' + + 'leaves one band per class, i.e. provenance-only priority.', + { unit: 'ms' } + ), + urgentMaxShare: option( + 0.2, + 'Ceiling on the fraction of one claim batch the `urgent` lane may take.\n\n' + + 'A DRAIN-SHARE cap, not an admission limit: urgent is served strictly first but never ' + + 'gets more than this share of a batch, so the lanes below always get at least ' + + '`1 - urgentMaxShare`. That is a hard structural bound with no token bucket and no ' + + 'admission bookkeeping to get wrong, and it is what stops a bulk "re-render everything ' + + 'now" from becoming a queue-wide outage. Earliest-due-first within the lane, so a flood ' + + 'is still served oldest-first.\n\n' + + '`0` disables the lane entirely. Any share small enough to floor to zero jobs still ' + + 'admits one, because a cap of zero would make an operator’s force-render silently never ' + + 'run.', + { min: 0, max: 1 } + ), + minShare: option( + { discovered: 0.1, cold: 0.02 }, + 'Minimum share of each claim batch reserved for a coarse class, keyed by name ' + + '(`urgent`, `submitted`, `discovered`, `cold`).\n\n' + + 'FLOORS, NOT FIXED SHARES, and the simulation is unambiguous about why. Strict lane ' + + 'priority starves the tail at EVERY capacity level including 100%, and its lag numbers ' + + 'look good precisely because it drops work — starvation is invisible in a lag metric. ' + + 'But fixed shares summing to 1.0 are also wrong: they leave nothing for the priority ' + + 'order to spend, and deviating from earliest-due-first only ever costs. Reserving a ' + + 'MINIMUM for the classes that need protecting and letting the rest compete in lane ' + + 'order measured far better in the tail — discovery reaching 71h instead of 133h at 50% ' + + 'capacity, 29h instead of 40h at 75%.\n\n' + + 'A reservation that goes unclaimed is released back to lane order within the same pass, ' + + 'so a floor for a class with nothing due costs nothing. Shares are of the batch, so they ' + + 'need not sum to 1; a sum above 1 is clamped by the batch size itself.' + ), + } + ), claimScanCap: option( 1000, 'Ceiling on schedule rows read per claim pass. A leased row keeps its overdue position in the ' + diff --git a/packages/plugin/src/http_handlers/bot_request.js b/packages/plugin/src/http_handlers/bot_request.js index 59d4321..776458c 100644 --- a/packages/plugin/src/http_handlers/bot_request.js +++ b/packages/plugin/src/http_handlers/bot_request.js @@ -383,7 +383,15 @@ async function renderNow({ url, cacheUrl, deviceType, cacheKey, request, routeSc // would strand: on this node the funnel lowers the floor in-process, and on any other node — // which is ~75% of keys, since schedule rows are residency-pinned — the guard band is what // keeps the row above the owner's floor and therefore claimable. - await writeSchedule(cacheKey, { nextRenderTime: currentMinuteMs(), fromSitemap: !!renderTarget?.sitemapUrl }); + // URGENT, because that is precisely what this path is: an authenticated request that says render + // this page now and wait for it. It is rate-limited by `queue.lanes.urgentMaxShare` like every + // other lane-0 write, so a spammed debug header degrades to slow render-nows rather than to a + // stalled queue. + await writeSchedule(cacheKey, { + nextRenderTime: currentMinuteMs(), + fromSitemap: !!renderTarget?.sitemapUrl, + urgent: true, + }); // Wake idle consumers now instead of waiting out the periodic status sync. Non-force // so a paused queue stays paused (the render then simply times out to the fallback). diff --git a/packages/plugin/src/metrics.js b/packages/plugin/src/metrics.js index 9464009..e37ca52 100644 --- a/packages/plugin/src/metrics.js +++ b/packages/plugin/src/metrics.js @@ -377,15 +377,22 @@ export const METRICS = Object.freeze({ 'paused = 1 when this node’s queue is paused at snapshot time, else 0 — makes "paused for hours" ' + 'alertable without polling the REST surface. ' + 'claim_scan_ms = claim-pass duration; watch the p95 trend, not the level. ' + + 'lane_granted = jobs granted per claim pass to each render lane (queue.lanes), with the lane in ' + + 'the method slot — `urgent`, `cold`, or `submitted/bN` / `discovered/bN` for the TTL bands. ' + + 'Lanes redistribute a fixed amount of work, so no total moves when they engage and this is the ' + + 'only series that shows the split. A lane sitting at zero while its backlog is non-empty is the ' + + 'failure to look for (a minShare floor too low, or a ttlBands list that put two very different ' + + 'cadences in one band); it is indistinguishable from health anywhere else. ' + 'reconcile_restored / reconcile_missing = schedule gaps repaired / found per sweep (they differ ' + 'when the per-sweep restore cap truncates the pass); expect zero — a steady rate means ' + 'something is CREATING gaps, and the reconcile log line names the URLs.', }, method: { - name: 'result (claim_scan_ms only)', - values: ['granted', 'empty', 'capped'], + name: 'result (claim_scan_ms) | lane (lane_granted)', + values: ['granted', 'empty', 'capped', 'urgent', 'submitted/bN', 'discovered/bN', 'cold'], description: - 'Only claim_scan_ms uses this slot: granted = jobs handed out, empty = nothing due, capped = the ' + + 'claim_scan_ms and lane_granted use this slot. On claim_scan_ms: ' + + 'granted = jobs handed out, empty = nothing due, capped = the ' + 'scan hit queue.claimScanCap without reaching a not-yet-due row (in-flight work is filling the ' + 'window). Every other series emits null here.', }, @@ -641,6 +648,20 @@ export const metrics = Object.freeze({ /** One claim pass's duration and how it ended — a queue_health series, so the queue reads in one scan. */ claimScan: (durationMs, result) => server.recordAnalytics(durationMs, 'queue_health', 'claim_scan_ms', result, null), + /** + * Jobs granted to one lane by one claim pass, labelled by lane. + * + * WITHOUT THIS THERE IS NO WAY TO ANSWER "IS PRIORITISATION WORKING". The whole mechanism is a + * redistribution of a fixed amount of work, so no total moves when it engages — only the split + * does, and the split is invisible in every existing series. A lane whose share collapses to zero + * while it still has due rows is the failure mode (a floor set too low, a band list that put two + * cadences in one lane), and it looks exactly like health from the outside. + * + * Emitted per lane per pass rather than per job: a pass grants at most `maxClaimLimit` jobs across + * a handful of lanes, so this is a few emits per claim rather than one per render. + */ + claimLane: (granted, laneLabel) => server.recordAnalytics(granted, 'queue_health', 'lane_granted', laneLabel, null), + /** One origin proxy on the serve path: time to response headers, status, and why. */ originFetch: (durationMs, statusCode, reason) => server.recordAnalytics(durationMs, 'origin_fetch', statusCode, reason, null), diff --git a/packages/plugin/src/resources/PrerenderAdmin.js b/packages/plugin/src/resources/PrerenderAdmin.js index e08ab89..dda9911 100644 --- a/packages/plugin/src/resources/PrerenderAdmin.js +++ b/packages/plugin/src/resources/PrerenderAdmin.js @@ -36,7 +36,8 @@ * POST /prerender_admin/explain { url, deviceType } super_user * POST /prerender_admin/schedule { cacheKey } -> local row super_user * POST /prerender_admin/queue { scope, paused } | super_user - * { action: 'reset-claim-floor' } + * { action: 'reset-claim-floor' } | + * { action: 'restamp-lanes', limit?, force? } * POST /prerender_admin/revalidate { url, deviceType } super_user * POST /prerender_admin/reconcile start a repair sweep super_user * POST /prerender_admin/sweep-orphans { dryRun?, maxDeletes? } super_user @@ -110,7 +111,7 @@ import { validateOverride, writeOverrides, } from '../util/configOverride.js'; -import { floorState, leaseInfo, minuteOf, writeSchedule } from '../util/renderSchedule.js'; +import { floorState, leaseInfo, minuteOf, restampLanes, writeSchedule } from '../util/renderSchedule.js'; import { mergeBreadthRow, finalizeBreadth } from '../util/crawlStats.js'; import { clampRange, readAnalyticsWindow } from '../util/analyticsRead.js'; import { decode } from '../util/contentEncoding.js'; @@ -1065,7 +1066,11 @@ export class PrerenderAdmin extends Resource { const nextRenderTime = currentMinuteMs(); // The write is residency-routed, so this reaches the owning node from any node — and it // goes through the funnel, which lowers this node's claim floor to cover it. - await writeSchedule(cacheKey, { nextRenderTime, fromSitemap: !!target.sitemapUrl }); + // URGENT. This is the admin "render this now" action — an explicit operator statement, which is + // exactly what lane 0 is for, and what retires the `nextRenderTime = 1` trick this endpoint used + // to be an alternative to. Bounded by `queue.lanes.urgentMaxShare`, so a bulk re-render cannot + // take more than that fraction of any claim batch however many rows it files here. + await writeSchedule(cacheKey, { nextRenderTime, fromSitemap: !!target.sitemapUrl, urgent: true }); // `claim` reads a node-local flag, so waking consumers only helps on the node that owns // the row. When another node owns it, what makes the row claimable there is the CLAIM @@ -1520,6 +1525,16 @@ export class PrerenderAdmin extends Resource { return json(await RenderQueue.resetClaimFloor()); } + // The lane migration. BOUNDED AND REPEATABLE rather than a background walk with a progress row: + // a restamped row leaves the range the pass queries, so calling this until `done` makes + // progress with no cursor to go stale and no partial state to resume from. Call it in a loop; + // `done` is false while a pass fills its window, which is what says more remains. + if (data?.action === 'restamp-lanes') { + const limit = Number.isFinite(data.limit) && data.limit > 0 ? Math.min(data.limit | 0, 50_000) : 5_000; + const result = await restampLanes({ limit, force: data.force === true }); + return json(result, result.error ? 409 : 200); + } + const scope = data?.scope ?? server.hostname; const paused = data?.paused; diff --git a/packages/plugin/src/resources/RenderQueue.js b/packages/plugin/src/resources/RenderQueue.js index 562d5bb..084d775 100644 --- a/packages/plugin/src/resources/RenderQueue.js +++ b/packages/plugin/src/resources/RenderQueue.js @@ -384,7 +384,15 @@ export class RenderQueue extends Resource { // path costs one atomic load and moves the floor not at all. That is load-bearing: a // lowering on every completed render would rewind the floor to the current minute // continuously and the whole 14× seek win would evaporate. - await writeSchedule(cacheKey, { nextRenderTime, fromSitemap: !!renderTarget.sitemapUrl }); + // `interval` — the DEMAND-RESOLVED cadence, not `base` — is what bands this row into a lane. + // This is the only writer that knows the ladder's answer, so it is the one that has to + // pass it: `laneFor` with no interval files into the SLOWEST band, so a promoted catalog + // page would sit in the 48h lane and the promotion would be undone by the scheduler. + await writeSchedule(cacheKey, { + nextRenderTime, + fromSitemap: !!renderTarget.sitemapUrl, + renderInterval: interval, + }); // Persist the rung ONLY on an actual move. 'held' must not write even when the // stored field is absent — absence already resolves to the base ceiling, so writing @@ -724,7 +732,12 @@ export class RenderQueue extends Resource { `Retrying ${cacheKey} in ${Math.round(wait / 60000)}m (failure strike ${strikes}` + `${fromSitemap ? '' : ', non-sitemap'})` ); - await writeSchedule(cacheKey, { nextRenderTime, fromSitemap }); + // THE CADENCE, NOT THE BACKOFF, bands the lane — `wait` is how far this failure pushed the row + // out, `interval` is still what it renders at. And `cold`, which is the point of this branch: + // a row past the fast-retry lane has failed repeatedly, so it belongs in the lane that is + // floored rather than prioritized. Without it, a broad origin outage walks the whole corpus + // through the fast lanes at full priority while every render fails. + await writeSchedule(cacheKey, { nextRenderTime, fromSitemap, renderInterval: interval, cold: true }); return 'slow'; } @@ -754,6 +767,7 @@ export class RenderQueue extends Resource { await writeSchedule(cacheKey, { nextRenderTime: currentMinuteMs() + interval, fromSitemap: !!renderTarget.sitemapUrl, + renderInterval: interval, }); } @@ -800,6 +814,10 @@ export class RenderQueue extends Resource { pass.scanTruncated ? 'capped' : pass.jobs.length ? 'granted' : 'empty' ); + // One emit per lane the pass visited. `pass.lanes` is absent when lanes are off, so the series + // simply does not exist until the feature does — rather than reporting a single fabricated lane. + for (const laneResult of pass.lanes ?? []) metrics.claimLane(laneResult.granted, laneResult.label); + const jobs = []; let notOwnedHere = 0; diff --git a/packages/plugin/src/resources/Target.js b/packages/plugin/src/resources/Target.js index 3a95349..d76054f 100644 --- a/packages/plugin/src/resources/Target.js +++ b/packages/plugin/src/resources/Target.js @@ -108,6 +108,10 @@ export class Target extends TargetTable { ? nextRenderTime : getInitialRenderTime(cacheKey, interval), fromSitemap, + // Bands the lane. Route > stored > default here; the demand rung does not exist yet for a + // brand-new target, so the first cycle bands at the route's ceiling and the post-render + // write corrects it. + renderInterval: interval, })) ); @@ -200,6 +204,13 @@ export class Target extends TargetTable { cacheKey, nextRenderTime: recheckAt, fromSitemap: !!existing?.sitemapUrl, + // COLD. A suppressed target has said it does not want to be indexed; its recheck is + // speculative work, and it must not compete for capacity with pages that are actually + // served. This is the single biggest slice of avoidable priority in the corpus — a + // recheck is scheduled for every suppressed URL on `suppression.recheckInterval` + // forever — and the cold lane is where it belongs. + cold: true, + renderInterval: resolveRenderInterval(url, existing?.renderInterval), })) ), ...cacheKeysOf(url).map((cacheKey) => PrerenderedPage.delete(cacheKey)), @@ -304,7 +315,18 @@ export class Target extends TargetTable { // that changes nothing. Hoisting the lowering out of the loop would mean carrying the // batch's rows in memory to no measurable end. await writeSchedules( - cacheKeysOf(url).map((cacheKey) => ({ cacheKey, nextRenderTime, fromSitemap: !!sitemapUrl })) + cacheKeysOf(url).map((cacheKey) => ({ + cacheKey, + nextRenderTime, + fromSitemap: !!sitemapUrl, + // Route-resolved, because this projection deliberately carries only `url` and + // `sitemapUrl` (see the note above on keeping the scan narrow). It is one term short of + // the post-render write — a stored `changefreq` interval is invisible here — and the + // consequence is bounded: the row is banded at its route's cadence for one cycle and + // re-banded correctly when it renders. Passing nothing would be far worse, since that + // bands every revalidated row into the SLOWEST lane. + renderInterval: resolveRenderInterval(url, null), + })) ); }, }); diff --git a/packages/plugin/src/util/invalidationReenqueue.js b/packages/plugin/src/util/invalidationReenqueue.js index d36d09e..052010b 100644 --- a/packages/plugin/src/util/invalidationReenqueue.js +++ b/packages/plugin/src/util/invalidationReenqueue.js @@ -316,7 +316,16 @@ export const accelerateHeal = async ({ url, cacheKey, invalidatedBy }) => { // schedule row we just read: `put` REPLACES the record, and the target is the field's source of // truth, so this self-corrects a row whose flag went stale (same choice as the reschedule path). await writeSchedules( - eligible.map((row) => ({ cacheKey: row.cacheKey, nextRenderTime: dueAt, fromSitemap: !!target.sitemapUrl })) + eligible.map((row) => ({ + cacheKey: row.cacheKey, + nextRenderTime: dueAt, + fromSitemap: !!target.sitemapUrl, + // Already resolved above for the eligibility arithmetic, so banding the lane costs nothing + // here. NOT urgent: an invalidation accelerates a page's cadence, it is not an operator + // asking for one page now, and routing a bulk epoch through lane 0 would spend the whole + // urgent share on it. + renderInterval: interval, + })) ); } catch (e) { logger.error(e, `[prerender] could not accelerate ${cacheKey} after an invalidation`); diff --git a/packages/plugin/src/util/laneFloor.js b/packages/plugin/src/util/laneFloor.js new file mode 100644 index 0000000..22c190a --- /dev/null +++ b/packages/plugin/src/util/laneFloor.js @@ -0,0 +1,196 @@ +/** + * ONE CLAIM WATERMARK PER LANE, over a shared buffer of its own. + * + * The floor is the entire performance story of the claim path, and #80 re-measured it per lane: + * three lanes interleaved in one index read in 0.29-0.32 ms each WITH a watermark, and 3.46 ms + * WITHOUT one. Interleaving is free; the watermark is the whole win. So a lane that has no + * watermark of its own is a lane that seeks from the absolute minimum of its slice on every pass, + * and it degrades the same way the unfloored scan did before v0.34.0 — 0.36 ms to 6.25 ms over + * 40,000 reschedules, linear, and it did not recover when the churn stopped. + * + * ── WHY A SEPARATE BUFFER RATHER THAN MORE HEADER IN `renderLease.js` ─────────────────────────── + * + * Because that module is a data structure with a probe protocol, an expiry boundary, a release + * grace and an eight-rule CAS contract, all pinned by tests that run against a bare + * `new ArrayBuffer()`. Widening its header changes the offset of every slot, and the named shared + * buffer is SIZED BY ITS FIRST ALLOCATION — so a worker generation that allocated the old layout + * hands the new code a correctly-sized-looking view with everything shifted. That is silent memory + * corruption in the module that decides which URLs render. + * + * A second named buffer cannot do that. It is sized independently, it is absent-or-present rather + * than subtly-misaligned, and the lease table keeps every invariant it already has. The cost is one + * more `getUserSharedBuffer` key per node. + * + * ── THE FLOOR RULE IS THE SAME RULE, PER LANE ────────────────────────────────────────────────── + * + * Read `util/renderSchedule.js`'s module comment for why it is what it is; nothing about it changes + * here except its scope. Per lane: + * + * floor_new = the due minute of the FIRST DUE ROW THAT LANE'S PASS OBSERVED — granted, skipped as + * already-leased, or refused for any reason. If it observed no due row at all: + * `nowMinute - guard`. + * + * Stated per lane rather than globally because the hazard is per lane too: a wedged row pins its OWN + * lane and nothing else. That is a strict improvement on the single global floor, where one + * permanently-failing product page pinned the scan position for the homepage as well. + * + * ── MINUTES, DECODED ─────────────────────────────────────────────────────────────────────────── + * + * A stored floor is a DUE MINUTE, never an encoded `nextRenderTime`. The lane's seek bound is + * rebuilt as `lane * LANE_STRIDE + floorMinute * MINUTE` at query time. Storing the encoded form + * would make floors incomparable across lanes — and "which lane is furthest behind" is the question + * both the console and the unpin hatch ask. + * + * NO DEPENDENCIES beyond the hash, for the same reason `renderLease.js` has none: this is a data + * structure, and `test/laneFloor.test.js` drives every CAS rule against a plain ArrayBuffer with an + * injected clock rather than against a mock of Harper. + */ + +import { lease64 } from './hash.js'; + +/** Per lane: [floorMinute, pinLo, pinHi, pinSinceSec]. */ +const L_FLOOR = 0; +const L_PIN_LO = 1; +const L_PIN_HI = 2; +const L_PIN_SINCE = 3; +const LANE_INT32 = 4; + +/** + * Pin timestamps are Int32 SECONDS relative to this constant, matching `renderLease.js`: raw epoch + * seconds overflow an Int32 in 2038, and a baked-in constant means two workers can never disagree + * about what a stored number means. + */ +export const LANE_EPOCH_SEC = 1_700_000_000; + +export const LANE_FLOOR_SAB_KEY = 'prerender/lane-floors'; + +/** Byte size of a lane-floor buffer for `lanes` lanes. */ +export const laneFloorBufferBytes = (lanes) => LANE_INT32 * 4 * Math.max(1, lanes | 0); + +/** How many lanes a buffer of this size holds. */ +export const laneFloorLanesIn = (byteLength) => Math.max(1, Math.floor(byteLength / (LANE_INT32 * 4))); + +const toSec = (ms) => Math.round(ms / 1000) - LANE_EPOCH_SEC; +const fromSec = (sec) => (sec + LANE_EPOCH_SEC) * 1000; + +/** + * @param {object} opts + * @param {ArrayBuffer} opts.buffer shared across the node's workers + * @param {number} [opts.lanes] lane count; clamped to what the buffer actually holds + * @param {() => number} [opts.now] injected clock, LATE-BOUND by the caller (see `leaseTable`) + */ +export const createLaneFloors = ({ buffer, lanes = laneFloorLanesIn(buffer.byteLength), now = Date.now } = {}) => { + const i32 = new Int32Array(buffer); + // Clamped to the buffer, never trusted from the argument: indexing past a short buffer is silent + // corruption, whereas deriving the count from the buffer we actually got is merely fewer lanes — + // and `renderSchedule.js` logs loudly when the two disagree. + const laneCount = Math.max(1, Math.min(lanes | 0, laneFloorLanesIn(buffer.byteLength))); + const base = (lane) => Math.min(Math.max(0, lane | 0), laneCount - 1) * LANE_INT32; + + /** + * The floor to seek this lane from, clamped `guard` minutes behind now on EVERY read. + * + * The guard is what makes a "render this URL now" write safe from any node with no cross-node + * coordination: schedule rows are residency-pinned, so most such writes are issued by a node that + * cannot lower the owner's floor — but they are written at the current minute, and every node + * holds its floor behind that by construction. + */ + const readFloorMinute = (lane, nowMinute, guard = 0) => { + const stored = Atomics.load(i32, base(lane) + L_FLOOR); + // A zero floor means NO FLOOR (seek the lane's absolute minimum), which is what a fresh buffer + // and a deliberate reset both mean. Clamping it up to `nowMinute - guard` would silently turn + // "re-derive from the bottom" into "skip everything older than the guard band" — i.e. it would + // strand exactly the rows a reset exists to recover. + if (stored <= 0) return 0; + return Math.min(stored, Math.max(0, nowMinute - guard)); + }; + + /** + * CAS-min: lower this lane's floor to cover `minute`, or leave it alone. + * + * The high-volume caller is every completed render writing `now + interval`, which is ABOVE the + * floor and therefore costs one atomic load and changes nothing. That negative half is where the + * win lives — a lowering on every render would rewind the floor continuously. + */ + const lowerFloorTo = (lane, minute) => { + const at = base(lane) + L_FLOOR; + const target = Math.max(0, Math.floor(minute)); + for (;;) { + const current = Atomics.load(i32, at); + // 0 already means unbounded, so nothing is lower. + if (current !== 0 && current <= target) return false; + if (Atomics.compareExchange(i32, at, current, target) === current) return true; + } + }; + + /** + * Advance this lane's floor from the value the pass started at to what it observed. ABANDONS on + * conflict rather than retrying: a conflict means a funnel write lowered the floor for a row this + * pass never saw, and re-advancing over it would strand that row. The next pass re-derives. + */ + const advanceFloor = (lane, from, to) => { + const at = base(lane) + L_FLOOR; + const target = Math.max(0, Math.floor(to)); + if (target <= from) return false; + return Atomics.compareExchange(i32, at, from, target) === from; + }; + + const resetFloor = (lane) => Atomics.store(i32, base(lane) + L_FLOOR, 0); + + const resetAllFloors = () => { + for (let lane = 0; lane < laneCount; lane++) resetFloor(lane); + }; + + /** + * Record which key is holding this lane's floor and return how long it has held it, in ms. + * `null` clears the pin — a pass that found nothing due must not leave a stale pin ageing forever. + * + * The key is stored as its 64-bit hash rather than its text, because the buffer is fixed-width; + * the CALLER reports the readable key from its own pass result. The hash exists only to answer + * "is this the same row as last time". + */ + const notePinnedBy = (lane, cacheKey) => { + const b = base(lane); + if (!cacheKey) { + Atomics.store(i32, b + L_PIN_LO, 0); + Atomics.store(i32, b + L_PIN_HI, 0); + Atomics.store(i32, b + L_PIN_SINCE, 0); + return 0; + } + const { lo, hi } = lease64(cacheKey); + const nowMs = now(); + const sameLo = Atomics.load(i32, b + L_PIN_LO) === lo; + const sameHi = Atomics.load(i32, b + L_PIN_HI) === hi; + if (sameLo && sameHi) { + const since = Atomics.load(i32, b + L_PIN_SINCE); + return since === 0 ? 0 : Math.max(0, nowMs - fromSec(since)); + } + Atomics.store(i32, b + L_PIN_LO, lo); + Atomics.store(i32, b + L_PIN_HI, hi); + Atomics.store(i32, b + L_PIN_SINCE, toSec(nowMs)); + return 0; + }; + + /** Every lane's floor minute and pin age — what the console and the unpin hatch read. */ + const snapshot = (nowMs = now()) => + Array.from({ length: laneCount }, (_, lane) => { + const b = base(lane); + const since = Atomics.load(i32, b + L_PIN_SINCE); + return { + lane, + floorMinute: Atomics.load(i32, b + L_FLOOR), + pinnedForMs: since === 0 ? 0 : Math.max(0, nowMs - fromSec(since)), + }; + }); + + return { + laneCount, + readFloorMinute, + lowerFloorTo, + advanceFloor, + resetFloor, + resetAllFloors, + notePinnedBy, + snapshot, + }; +}; diff --git a/packages/plugin/src/util/laneRestamp.js b/packages/plugin/src/util/laneRestamp.js new file mode 100644 index 0000000..b8d254a --- /dev/null +++ b/packages/plugin/src/util/laneRestamp.js @@ -0,0 +1,157 @@ +/** + * THE ONE-TIME MIGRATION, and the reason it is needed at all. + * + * A lane is DERIVED AT WRITE TIME, which is what makes a config change retroactive with no sweep of + * the corpus — every row picks up the new banding on its next render. That property is real and it + * is why the design avoids a lane column. It also has one hole, and the hole is exactly the + * situation lanes exist to fix: + * + * A ROW THAT IS STUCK NEVER GETS A NEXT WRITE. Nothing rewrites a schedule row except its own + * render result, so a homepage sitting behind three days of backlog cannot be promoted by a + * write-time rule — it is not being written. Deploying lanes and waiting would mean waiting for + * the very backlog the lanes are supposed to clear. + * + * So enabling lanes on an existing corpus takes one pass that rewrites each due time into its + * derived lane. It changes NO due time: `dueAt` is preserved exactly and only the high bits move, + * so nothing renders sooner or later than it would have — the pass changes the ORDER and nothing + * else. That is also what makes it safe to run on a live node. + * + * ── WHAT IT CAN AND CANNOT DERIVE ────────────────────────────────────────────────────────────── + * + * The row carries `fromSitemap`; the cadence is route-resolved from the URL half of the cache key. + * It deliberately does NOT read the Target, which would be one cross-database point read per row on + * an 814k-target corpus. Two consequences, both bounded and both self-correcting on each row's next + * render: + * + * - A stored `changefreq` interval or a demand-ladder rung is invisible, so a catalog page the + * ladder promoted to 6h is stamped at its route's 24h ceiling and lands one band slower than it + * belongs. It re-bands correctly the first time it renders. + * - `cold` cannot be derived — suppression state and strike counts live on the Target — so a + * suppressed URL is stamped into its provenance lane rather than into `cold`. It moves to `cold` + * on its next recheck, which is the write that knows. + * + * Being one band optimistic for one cycle is the right direction to be wrong in: the alternative is + * being pessimistic, and a row parked in a slow lane it does not belong in is a row that waits. + * + * ── WHY IT ONLY LOOKS AT LANE 0 ─────────────────────────────────────────────────────────────── + * + * Unencoded and "urgent" are the same number — that is what makes the encoding migration-free, and + * it means this pass cannot tell a row nobody has stamped yet from a row an operator deliberately + * put in lane 0. While `queue.lanes.enabled` is false no legitimate lane-0 row can exist (writes + * file lane 0 unconditionally and the claim path ignores lanes), so running the pass BEFORE flipping + * the switch makes the ambiguity disappear rather than managed. That ordering is the documented + * rollout, and this refuses to run once the switch is on unless explicitly forced. + * + * Selecting on lane 0 is also what makes the pass idempotent and resumable with no cursor: a + * restamped row leaves the range being queried, so calling it repeatedly makes progress and + * eventually finds nothing. There is no progress row to go stale and no partial state to resume + * from. + */ + +import { config } from '../config.js'; +import { CacheKey } from './cacheKey.js'; +import { LANE_STRIDE, laneFor, laneLabel } from './renderLane.js'; +import { resolveRenderInterval } from './routeClass.js'; +import { numberOf } from './time.js'; + +/** + * One bounded pass. Returns what it did, and `done` when a pass examined rows and found nothing + * left to move. + * + * ALL I/O INJECTED, for the same reason `runClaimPass` takes its search as an argument: the lane + * derivation and the batching are the parts worth testing, and they should be testable without a + * database. + * + * @param {object} io + * @param {(opts: {limit: number}) => AsyncIterable} io.searchUnstamped lane-0 rows, ascending + * @param {(cacheKey: string, row: object) => Promise} io.writeRow the funnel's batch write + * @param {number} [limit] ceiling on rows examined in this pass + */ +export const restampPass = async ({ searchUnstamped, writeRow, limit = 5000 } = {}) => { + const rows = []; + // Drained before any write, exactly like the claim pass: Harper's long-transaction monitor aborts + // a transaction that has pending writes when it fires, and a cursor left open across writes is + // that shape. No `break` either — an abandoned iterator leaves its read transaction unreleased. + for await (const row of searchUnstamped({ limit })) rows.push(row); + + const byLane = new Map(); + let examined = 0; + let restamped = 0; + let skipped = 0; + + for (const row of rows) { + examined++; + const dueAt = numberOf(row.nextRenderTime); + // A row with no usable due time is left alone. It is not this pass's job to invent one, and + // re-filing it under a lane would move a broken row without fixing it — `util/reconcile.js` and + // the backlog snapshot are what report those. + if (!Number.isFinite(dueAt)) { + skipped++; + continue; + } + + const url = CacheKey.extractUrl(row.cacheKey); + const fromSitemap = !!row.fromSitemap; + const renderInterval = resolveRenderInterval(url, null); + const lane = laneFor({ fromSitemap, renderInterval }); + + // Lane 0 rows that DERIVE to lane 0 are already where they belong, so no write. On this corpus + // that is only the fastest submitted band when it is also the first band, so it is a small + // share — but writing them anyway would double the pass's write volume for no change. + if (lane === 0) { + skipped++; + continue; + } + + if (!byLane.has(lane)) byLane.set(lane, []); + byLane.get(lane).push({ cacheKey: row.cacheKey, nextRenderTime: dueAt, fromSitemap, renderInterval }); + restamped++; + } + + // Grouped by lane so each batch lowers ONE lane's watermark once, with that lane's minimum. A + // single mixed batch would lower whichever lane owned the earliest row and leave the others' + // watermarks above rows they now have to find — which is the stranding the funnel exists to + // prevent, and it would be invisible until those lanes reported nothing due. + for (const rowsForLane of byLane.values()) await writeRow(rowsForLane); + + return { + examined, + restamped, + skipped, + lanes: [...byLane.entries()].map(([lane, rowsForLane]) => ({ + lane, + label: laneLabel(lane), + rows: rowsForLane.length, + })), + // `examined < limit` is what proves the range is exhausted rather than merely capped: a full + // window says only that more may remain. A pass that restamped nothing AND filled its window + // is not done — it means every row it saw was already correct, and the next pass starts past + // them only because they are no longer in the queried range. + done: examined < limit, + }; +}; + +/** The one-condition query for rows nobody has stamped: everything below the first lane boundary. */ +export const unstampedQuery = (limit) => ({ + // ONE CONDITION, matching the claim path's reasoning: a two-sided range on this index degrades to + // a post-filter (measured 1,128-2,977 ms). `less_than` alone is a clean index range, and it is + // sufficient because lane 0 is the bottom of the space. + conditions: [{ attribute: 'nextRenderTime', comparator: 'less_than', value: LANE_STRIDE }], + sort: { attribute: 'nextRenderTime' }, + // ARRAY select — a string `select` returns the bare scalar rather than a record. + select: ['cacheKey', 'nextRenderTime', 'fromSitemap'], + limit, +}); + +/** Whether a restamp is allowed to run right now, and why not if it is not. */ +export const restampGuard = ({ force = false } = {}) => { + if (force || !config.queue.lanes.enabled) return { allowed: true }; + return { + allowed: false, + reason: + 'queue.lanes.enabled is true, so a lane-0 row can no longer be told apart from a row an operator ' + + 'deliberately marked urgent — this pass would demote them. Run the restamp BEFORE enabling lanes ' + + '(that is the documented rollout), or pass force: true if you accept that any in-flight urgent ' + + 'request loses its priority and renders on its normal cadence instead.', + }; +}; diff --git a/packages/plugin/src/util/reconcile.js b/packages/plugin/src/util/reconcile.js index 3a9935b..ff6db34 100644 --- a/packages/plugin/src/util/reconcile.js +++ b/packages/plugin/src/util/reconcile.js @@ -121,6 +121,9 @@ export const reconcileSchedules = async ({ // repaired row would be filed one step behind the floor and this sweep would restore // rows into the very silent gap it exists to close. nextRenderTime: getInitialRenderTime(cacheKey, resolveRenderInterval(target.url, target.renderInterval)), + // Bands the restored row's lane. Same resolution the row's own reschedule will use, so a + // repair does not park a fast route in the slow lane until its next render. + renderInterval: resolveRenderInterval(target.url, target.renderInterval), fromSitemap: !!target.sitemapUrl, }); stats.restored++; diff --git a/packages/plugin/src/util/renderLane.js b/packages/plugin/src/util/renderLane.js new file mode 100644 index 0000000..2c54f0f --- /dev/null +++ b/packages/plugin/src/util/renderLane.js @@ -0,0 +1,349 @@ +/** + * RENDER LANES — priority as a range of the `nextRenderTime` index, not as a second index. + * + * `claim` is strictly `nextRenderTime`-ascending, so under any capacity deficit the queue serves + * whatever is oldest-due. Two measurements say that is the wrong order, and prerender-plugin#80 + * carries both: + * + * PROVENANCE. During a multi-hour production backlog, 239,090 of 521,929 overdue rows (~46%) + * were bot-discovered rather than sitemap-submitted. Half the render capacity was going to pages + * the site owner never submitted while submitted pages aged out of their SWR window. + * + * TTL-BLINDNESS, which is the sharper one. Absolute due time treats a 1h-TTL homepage 3h overdue + * exactly like a 48h-TTL product page 3h overdue. The first is 300% stale, the second 6%. So the + * shortest-TTL route is structurally the most damaged, and it is damaged EVEN AT FULL CAPACITY: + * simulated over the real corpus, home sits at 4.78x its own TTL at 100% capacity and 48.83x at + * 50%, against 1.08x / 2.00x for product. + * + * ── WHY NOT RANK BY RELATIVE LATENESS, WHICH IS THE OBVIOUS FIX ───────────────────────────────── + * + * Because it cannot be an ORDER. `relativeLateness(t) = (t - dueAt) / interval` is linear in `t` + * with slope `1/interval`, so two rows with different intervals have different slopes and cross + * exactly once. No stored key can express an order that changes with the clock, which means no + * index can serve it. + * + * Re-ranking inside the claim window does not rescue it either, and this is the part that looks + * like it should work. The pass reads a bounded window from the claim floor — ~140 rows in + * production — and that window is ALREADY an EDF prefix. A 1h page sitting 600 minutes down the + * queue never appears in it to be re-ranked at all. Widening the window does not help either: the + * window is anchored at the OLDEST due time, and under a deep backlog every row in it is ancient. + * A homepage that is two of its own cadences late is numerically nowhere near the head of an index + * whose head is three days overdue. + * + * So relative lateness is the RATIONALE for which lane a row belongs in, and never the comparator. + * Lanes are cut so that intervals inside one are similar by construction; EDF on `dueAt` within a + * lane is then both index-backable and a good approximation of relative lateness — and EDF is + * provably optimal for maximum lateness, so any deviation inside a lane only ever costs. + * + * ── THE ENCODING ──────────────────────────────────────────────────────────────────────────────── + * + * nextRenderTime = lane * STRIDE + dueAtMs + * + * One column, the existing index, no schema change and no second index. Measured (#80, + * `21-duerank.mjs`, 200k rows): three lanes interleaved in ONE index, each with its own watermark, + * read in 0.29-0.32 ms per lane — interleaving costs nothing. The same lane with its watermark + * reset to zero costs 3.46 ms, so THE WATERMARK IS THE ENTIRE WIN, not the separation. And a + * second index is not free: the existing secondary index is already 39-48% of reschedule wall + * clock, so a per-lane index or table roughly doubles the hot write. + * + * Lower value is claimed first, so lane order IS priority order and `urgent` needs no encoding at + * all — the hottest thing an operator can ask for is the cheapest to express. + * + * `STRIDE = 2^42` ms is ~139 years, comfortably above any real timestamp (now ~1.79e12), and + * Harper's `Long` is 52-bit-safe (9.007e15), so it yields 2,048 lanes. This module uses at most a + * couple of dozen. Lanes are effectively free, which is why the taxonomy below can afford to band + * by TTL rather than economizing. + * + * ── WHY NOT THE OTHER ENCODINGS ───────────────────────────────────────────────────────────────── + * + * An additive OFFSET on the due time is too weak: a backlog deeper than the offset absorbs it and + * the lane goes invisible once everything is overdue — which is the steady state this exists for. + * + * LOW-ORDER bits are free (due times are minute-floored, so ~59,999 ms per minute are unused) and + * useless: they only break ties within a single minute, and a 600-minute backlog ignores them. + * + * TIME-BUCKETING (bucket-major, lane-minor) works and keeps values valid timestamps, but it + * spends due-time precision — so it breaks exactly the 1h route this exists to fix — and the + * bucket must be no larger than `swrTtl`. + * + * ── AND WHY LANE 0 IS NOT THE DEFAULT ─────────────────────────────────────────────────────────── + * + * Every row written before this existed is unencoded, i.e. numerically in lane 0. That is what + * makes the encoding migration-free, and it is also a trap: lane 0 is `urgent`, so on the first + * deploy the ENTIRE corpus reads as urgent, and `lanes.urgentMaxShare` would then ration the whole + * queue down to a fifth of capacity. The rollout is therefore two steps and the switch ships off — + * see `queue.lanes.enabled`. While it is off, nothing here is consulted and the claim pass is + * byte-for-byte what it was. + */ + +import { config } from '../config.js'; +import { numberOf } from './time.js'; + +/** + * The lane multiplier. 2^42 ms ~= 139 years: high enough that no real `dueAt` can reach into the + * next lane, low enough that 2,048 lanes fit inside Harper's 52-bit-safe `Long`. + * + * A power of two rather than a round decimal so `Math.floor(v / STRIDE)` and `v % STRIDE` are exact + * at every magnitude a `Long` can hold — a decimal stride makes the modulo drift in the last bits + * once values pass 2^53, and a due time recovered one millisecond wrong is a due time. + */ +export const LANE_STRIDE = 2 ** 42; + +/** Ceiling on lane count implied by `LANE_STRIDE` against a 52-bit-safe `Long`. */ +export const MAX_LANES = Math.floor(Number.MAX_SAFE_INTEGER / LANE_STRIDE); + +/** + * The COARSE classes, in priority order. These are the names an operator reasons about, the names + * that appear in logs and on the console, and the keys `queue.lanes.minShare` is written against. + * + * Named and ordered rather than a free-form integer on purpose: self-documenting in a log line, + * reviewable in a diff, and it cannot drift into arbitrary magic numbers the way a priority int + * does. + */ +export const URGENT = 'urgent'; +export const SUBMITTED = 'submitted'; +export const DISCOVERED = 'discovered'; +export const COLD = 'cold'; + +/** Coarse classes in priority order — index into this is the class's rank, not its lane. */ +export const LANE_CLASSES = [URGENT, SUBMITTED, DISCOVERED, COLD]; + +/** + * TTL bands, ascending, from config. A row's band is the first entry its interval does not exceed; + * an interval past every entry lands in the overflow band, so `bands.length + 1` bands exist. + * + * WHY BAND AT ALL, given the coarse classes above already fix provenance: because a class whose + * intervals are NOT similar reproduces TTL-blindness inside itself. `submitted` at this deployment + * spans 1h to 48h, so EDF within it is precisely the order we are trying to leave behind, and the + * homepage would go on losing — to sitemap-submitted product pages instead of discovered ones. + * Banding is what makes "EDF within a lane approximates relative lateness" true rather than + * aspirational. + * + * `discovered` is banded too, which is a deliberate DEVIATION from the simulation in #80 — that + * modelled discovery as a single lane. Banding it can only reduce within-lane lateness and it does + * not touch the inter-lane floors the simulation actually measured, so the measured tail numbers + * still bound this. It costs nothing: lanes are effectively free. + */ +const bands = () => { + const raw = config.queue.lanes.ttlBands ?? []; + // Sorted, de-duplicated and filtered here rather than trusted from config: the band index IS + // part of the stored key, so an unsorted list would assign a shorter interval to a later lane + // and quietly invert the ordering it exists to create. + return [...new Set(raw.filter((ms) => Number.isFinite(ms) && ms > 0))].sort((a, b) => a - b); +}; + +/** How many bands each banded class occupies (the configured cuts, plus the overflow band). */ +export const bandCount = () => bands().length + 1; + +/** Which band an interval falls in: 0 for the fastest, `bandCount() - 1` for the overflow. */ +export const bandOf = (intervalMs) => { + const cuts = bands(); + const ms = Number(intervalMs); + // A missing or unusable interval takes the SLOWEST band, not the fastest. It is the only safe + // direction: an absent interval is an absent claim to urgency, and defaulting to band 0 would + // let any row that lost its cadence jump ahead of the homepage. + if (!Number.isFinite(ms) || ms <= 0) return cuts.length; + for (let i = 0; i < cuts.length; i++) if (ms <= cuts[i]) return i; + return cuts.length; +}; + +/** + * The lane layout, which is entirely determined by `bandCount()`: + * + * 0 urgent (unbanded — operator intent has no cadence argument) + * 1 .. B submitted, band 0 .. B-1 + * B+1 .. 2B discovered, band 0 .. B-1 + * 2B+1 cold (unbanded — a failing row's cadence is not the point) + * + * `urgent` and `cold` are deliberately unbanded. Urgent is an operator statement that outranks + * every cadence argument by construction, and banding it would let a slow-route urgent request lose + * to a fast-route one — which is not a thing an operator asked for. Cold is the opposite: a row that + * has failed repeatedly has no credible cadence to be judged against, and banding it would let a + * broken 1h route consume the floor reserved for the whole failing tail. + */ +export const laneCount = () => 2 * bandCount() + 2; + +const COLD_LANE = () => laneCount() - 1; + +/** + * The lane for a row, from the same stable inputs `resolveRenderInterval` uses. + * + * DERIVED, NEVER STORED — that is load-bearing. Resolving at write time from `sitemapUrl` presence, + * the route-resolved interval and the failure count means a config change (a new band cut, a route + * moved to a different cadence) is retroactive on each key's next render with NO sweep of the + * corpus. Storing a lane column would mean every such change needed one. + * + * @param {object} row + * @param {boolean} row.fromSitemap the target carries a `sitemapUrl` + * @param {number} row.renderInterval the EFFECTIVE cadence in ms (route > stored > default, + * demand rung applied) — the same number the due time was computed from + * @param {boolean} [row.urgent] operator intent; outranks everything + * @param {boolean} [row.cold] repeatedly failing, or never successfully rendered + */ +export const laneFor = ({ fromSitemap, renderInterval, urgent = false, cold = false } = {}) => { + if (urgent) return 0; + if (cold) return COLD_LANE(); + const band = bandOf(renderInterval); + return (fromSitemap ? 1 : 1 + bandCount()) + band; +}; + +/** The coarse class a lane belongs to — for logs, metrics dimensions and `minShare` lookup. */ +export const classOfLane = (lane) => { + const B = bandCount(); + if (lane <= 0) return URGENT; + if (lane >= COLD_LANE()) return COLD; + return lane <= B ? SUBMITTED : DISCOVERED; +}; + +/** A stable, readable label for one lane: `submitted/b0`, `discovered/b2`, `urgent`, `cold`. */ +export const laneLabel = (lane) => { + const klass = classOfLane(lane); + if (klass === URGENT || klass === COLD) return klass; + const B = bandCount(); + return `${klass}/b${(lane - 1) % B}`; +}; + +/** + * Encode a due time into a lane. The inverse of `dueAtOf`/`laneOf`. + * + * Clamped rather than throwing: a lane past the layout can only come from a config edit racing a + * write, and a row filed one lane too low still renders — whereas throwing here would fail a + * schedule write, which is the one outcome that loses a page. + */ +export const encodeDueAt = (dueAtMs, lane) => { + // `numberOf`, NOT `Number`: `Number(null)` is 0, and 0 is finite — so a bare coercion would turn + // an ABSENT due time into a real due time of 0, and a floor of zero means NO FLOOR — which puts + // the claim scan back to seeking the absolute index minimum. The value is returned UNCHANGED + // rather than normalized, so absence stays absence all the way to the row and every existing + // guard downstream still recognizes it. (A real 0 still encodes, and still unbounds the floor — + // that is the documented `nextRenderTime = 1` shape and it must keep working.) + const at = numberOf(dueAtMs); + if (!Number.isFinite(at)) return dueAtMs; + const bounded = Math.min(Math.max(0, lane | 0), MAX_LANES - 1); + return bounded * LANE_STRIDE + at; +}; + +/** The lane an encoded value sits in. An unencoded (pre-lanes) value reads as lane 0. */ +export const laneOf = (encoded) => { + const v = numberOf(encoded); + return Number.isFinite(v) && v > 0 ? Math.floor(v / LANE_STRIDE) : 0; +}; + +/** + * The due time inside an encoded value — what every caller that wants a TIMESTAMP must use. + * + * This is the one function whose absence at a call site is silent and expensive: an encoded value + * used as a date is a date ~139 years per lane in the future, so a served page's `expiresAt`, a + * console's "next render" column and the invalidation accelerator's `nextRenderTime - interval` + * arithmetic all read as plausible-but-wrong rather than as an error. The funnel decodes on the way + * out (`getScheduleRow`, the claim projection) so that in-plugin readers cannot forget; anything + * reading the exported REST surface directly has to call this itself. + */ +export const dueAtOf = (encoded) => { + const v = numberOf(encoded); + // Same rule as `encodeDueAt`: an unusable value is handed back exactly as it arrived, so a null + // stays a null rather than becoming a due time of zero somewhere downstream. + if (!Number.isFinite(v)) return encoded; + return v > 0 ? v % LANE_STRIDE : v; +}; + +/** Inclusive lower / exclusive upper bound of one lane's slice of the index. */ +export const laneRange = (lane) => ({ from: lane * LANE_STRIDE, to: (lane + 1) * LANE_STRIDE }); + +/** + * THE FAIRNESS ALLOCATOR — how one claim batch is divided between lanes. + * + * This is scheduler policy and it is deliberately NOT part of the ordering key. Keeping it out + * means it is tunable live with no rewriting of stored rows; any scheme that bakes fairness into + * the encoding needs the whole corpus re-encoded to change a bound. That is the strongest single + * argument for keeping ordering and fairness separate, and #80 makes it explicitly. + * + * Two findings from the simulation drive the shape, and both are load-bearing: + * + * STRICT PRIORITY STARVES THE TAIL AT EVERY CAPACITY LEVEL, INCLUDING 100%. Its lag numbers look + * excellent precisely BECAUSE it drops work — starvation is invisible in a lag metric. So lane + * order alone is not the policy. + * + * FLOORS BEAT FIXED SHARES. Fixed shares summing to 1.0 leave nothing for the priority order to + * spend, and since EDF is optimal for maximum lateness, deviating from it costs. Reserving a + * MINIMUM for the classes that need protecting and letting the rest compete in lane order is far + * better in the tail: discovery reaches 71 h instead of 133 h at 50% capacity, 29 h instead of + * 40 h at 75%. + * + * So: lanes are visited in priority order, `urgent` is capped by its DRAIN SHARE rather than by + * admission (a hard structural bound that needs no token bucket and no admission bookkeeping — lanes + * below it always get at least `1 - urgentMaxShare` of every batch), and the protected classes are + * guaranteed a floor that earlier lanes may not spend. + * + * A reservation that goes unclaimed is not lost: `topUp()` releases it back to lane order once every + * lane has had its turn, so a floor for a class with nothing due costs nothing. + */ +export const createLaneBudget = ({ grantLimit, urgentMaxShare = 0, minShare = {} } = {}) => { + const total = Math.max(0, grantLimit | 0); + // `Math.max(1, ...)` so a share small enough to floor to zero still admits one job: a cap of 0 + // would silently make the lane unreachable, which for `urgent` means an operator's force-render + // never runs and nothing says why. + const urgentCap = urgentMaxShare > 0 ? Math.max(1, Math.floor(total * urgentMaxShare)) : 0; + + const reserve = new Map(); + for (const [klass, share] of Object.entries(minShare)) { + const n = Math.floor(total * (Number(share) || 0)); + if (n > 0) reserve.set(klass, n); + } + + let spent = 0; + let releaseReservations = false; + const spentByClass = new Map(); + const spentIn = (klass) => spentByClass.get(klass) ?? 0; + + return { + get remaining() { + return Math.max(0, total - spent); + }, + + /** How many jobs `lane` may be granted right now. */ + allowanceFor(lane) { + const remaining = Math.max(0, total - spent); + if (remaining === 0) return 0; + const klass = classOfLane(lane); + + let allowance = remaining; + if (!releaseReservations) { + // Hold back whatever is still owed to the OTHER protected classes... + let heldBack = 0; + for (const [c, n] of reserve) { + if (c === klass) continue; + heldBack += Math.max(0, n - spentIn(c)); + } + // ...but never below this class's own outstanding reservation, or a class whose floor is + // the last thing left could be held back by its own siblings' floors and starve on the + // mechanism meant to protect it. + const own = Math.max(0, (reserve.get(klass) ?? 0) - spentIn(klass)); + allowance = Math.max(remaining - heldBack, Math.min(remaining, own)); + } + + if (klass === URGENT) allowance = Math.min(allowance, Math.max(0, urgentCap - spentIn(URGENT))); + return Math.max(0, Math.min(allowance, remaining)); + }, + + record(lane, granted) { + if (granted <= 0) return; + spent += granted; + const klass = classOfLane(lane); + spentByClass.set(klass, spentIn(klass) + granted); + }, + + /** + * Release every unclaimed reservation, for a second sweep in lane order. + * + * Needed because a floor is a MINIMUM, not an entitlement: a class with fewer due rows than + * its floor would otherwise leave that slice of the batch unspent while lanes above it had + * work. `urgentMaxShare` is deliberately NOT released — it is a cap, not a reservation, and + * the whole point of capping a drain share is that it holds even when nothing else wants the + * capacity. + */ + topUp() { + releaseReservations = true; + }, + }; +}; diff --git a/packages/plugin/src/util/renderSchedule.js b/packages/plugin/src/util/renderSchedule.js index d3c9757..4a6acfe 100644 --- a/packages/plugin/src/util/renderSchedule.js +++ b/packages/plugin/src/util/renderSchedule.js @@ -130,6 +130,19 @@ import { CacheKey } from './cacheKey.js'; import { resolveRenderInterval } from './routeClass.js'; import { MINUTE, numberOf } from './time.js'; import { LEASE_SAB_KEY, createLeaseTable, leaseBufferBytes, leaseSlotsIn } from './renderLease.js'; +import { LANE_FLOOR_SAB_KEY, createLaneFloors, laneFloorBufferBytes } from './laneFloor.js'; +import { restampGuard, restampPass, unstampedQuery } from './laneRestamp.js'; +import { + LANE_STRIDE, + classOfLane, + createLaneBudget, + dueAtOf, + encodeDueAt, + laneCount, + laneFor, + laneLabel, + laneOf, +} from './renderLane.js'; /** * The live lease table + claim floor, over one named buffer shared by every worker on this node. @@ -181,6 +194,61 @@ export const leaseTable = () => { return liveLeaseTable; }; +/** + * The per-lane claim watermarks, over a named buffer of their own. + * + * SEPARATE FROM THE LEASE BUFFER ON PURPOSE — see the module comment in `util/laneFloor.js`. In + * short: a named shared buffer is sized by its FIRST allocation, so widening the lease header would + * hand a new worker generation a correctly-sized-looking view of the old layout with every slot + * shifted, in the module that decides which URLs render. A second key cannot do that. + * + * Allocated on first use for the same reason `leaseTable` is: `laneCount()` reads + * `queue.lanes.ttlBands`, and module scope precedes `applyOptions`. + * + * SIZED FOR THE CEILING, NOT FOR THE CURRENT BAND LIST. `laneCount()` moves when an operator edits + * `ttlBands`, and the buffer cannot be resized within a process — so it is allocated at the maximum + * this build supports and only the first `laneCount()` entries are ever touched. That makes a live + * band-list edit a matter of which entries are USED rather than of reallocating shared memory, and + * it means a band added at runtime gets a zeroed (unbounded) floor, which is exactly right: a lane + * nothing has ever scanned should re-derive from the bottom of its slice. + */ +const MAX_SUPPORTED_LANES = 64; +let liveLaneFloors = null; + +export const laneFloors = () => { + if (liveLaneFloors) return liveLaneFloors; + const buffer = getSab(LANE_FLOOR_SAB_KEY, laneFloorBufferBytes(MAX_SUPPORTED_LANES)); + liveLaneFloors = createLaneFloors({ buffer, lanes: MAX_SUPPORTED_LANES, now: () => Date.now() }); + if (laneCount() > liveLaneFloors.laneCount) { + logger.error( + `[prerender] queue.lanes.ttlBands implies ${laneCount()} lanes but the lane-floor buffer holds ` + + `${liveLaneFloors.laneCount}. Lanes above that share the last watermark and will scan from a floor ` + + `that is not theirs. Reduce ttlBands or raise MAX_SUPPORTED_LANES.` + ); + } + return liveLaneFloors; +}; + +/** + * `runClaimPass` wants an object with a floor; this binds one lane's watermark into that shape. + * + * A thin adapter rather than teaching `laneFloor.js` the interface, so that module stays a data + * structure over lane INDICES and the coupling to the claim pass lives here, with the rest of the + * Harper-aware code. + */ +const floorsForLane = (lane) => { + const table = laneFloors(); + return { + readFloorMinute: (nowMinute, guard) => table.readFloorMinute(lane, nowMinute, guard), + advanceFloor: (from, to) => table.advanceFloor(lane, from, to), + resetFloor: () => table.resetFloor(lane), + notePinnedBy: (cacheKey) => table.notePinnedBy(lane, cacheKey), + // Deliberately no `recordPassOutcome`: `sawDue` / `earliestNotYetDue` drive the node's + // queue-status derivation, which is one answer for the whole node rather than one per lane. + // The orchestrator ORs the lanes' results and records that once, on the lease table. + }; +}; + // Resolved per call rather than destructured at module load, matching `util/reconcile.js` and // `util/backlogSnapshot.js`. This module is imported by almost everything (Target → Sitemap → // RenderQueue → the handlers), so a module-scope capture would make the import order of the whole @@ -201,7 +269,7 @@ const guardMinutes = () => Math.max(0, Math.round(config.queue.claimFloor.guard * 14× actually lives: a lowering on every completed render would rewind the floor to the current * minute continuously and the whole win would evaporate. */ -const lowerFloorFor = (nextRenderTime) => { +const lowerFloorFor = (nextRenderTime, lane = 0) => { // `numberOf`, not `Number`: `Number(null)` is 0, 0 is finite, and `lowerFloorTo(0)` means NO FLOOR // — so a single missing due time would silently put the scan back to seeking the absolute index // minimum, which is the degraded 6.25 ms seek this whole release exists to remove. A REAL 0 still @@ -213,6 +281,12 @@ const lowerFloorFor = (nextRenderTime) => { logger.warn(`[prerender] schedule write with a non-numeric nextRenderTime (${nextRenderTime}) — floor not lowered`); return; } + // BOTH, and the asymmetry matters. The lane watermark takes the DECODED minute (a lane's floor is + // a due minute, so floors stay comparable across lanes); the global floor takes the ENCODED value + // it would actually be compared against, because that is the number the un-laned seek uses. Both + // are lowered on every write regardless of which mode is live, so flipping `queue.lanes.enabled` + // never leaves the mode being switched TO with a floor above rows it now has to find. + laneFloors().lowerFloorTo(lane, minuteOf(dueAtOf(at))); leaseTable().lowerFloorTo(minuteOf(at)); }; @@ -240,14 +314,33 @@ const lowerFloorFor = (nextRenderTime) => { * in 10.7 ms, mean 0.021 ms, against residency pinned to a node that does not exist). v0.15.0 * assumed the read/write symmetry and wrapped these in a deadline that could never fire. */ -export const writeSchedule = async (cacheKey, { nextRenderTime, fromSitemap } = {}) => { +export const writeSchedule = async (cacheKey, { nextRenderTime, fromSitemap, renderInterval, urgent, cold } = {}) => { if (fromSitemap === undefined) { throw new Error(`writeSchedule(${cacheKey}) needs an explicit fromSitemap — put replaces the record`); } - await scheduleTable().put(cacheKey, { nextRenderTime, fromSitemap }); - lowerFloorFor(nextRenderTime); + const lane = laneToWrite({ fromSitemap, renderInterval, urgent, cold }); + const stored = encodeDueAt(nextRenderTime, lane); + await scheduleTable().put(cacheKey, { nextRenderTime: stored, fromSitemap }); + lowerFloorFor(stored, lane); }; +/** + * The lane a write should file into, or 0 when lanes are off. + * + * DERIVED HERE AND NOWHERE ELSE. Every schedule write in the plugin already funnels through this + * module, which is what makes "resolve the lane at write time" a single line rather than a + * discipline sixteen call sites have to remember — the same reason the floor lowering lives here. + * Deriving (rather than storing a lane column) is what makes a config change — a new band cut, a + * route moved to a different cadence — retroactive on each key's next render with no sweep. + * + * `renderInterval` is the EFFECTIVE cadence and it is what bands the row. A caller that does not + * have it hands in nothing, and `bandOf` puts the row in the SLOWEST band — deliberately, because + * an absent interval is an absent claim to urgency and the alternative would let any row that lost + * its cadence jump ahead of the homepage. + */ +const laneToWrite = ({ fromSitemap, renderInterval, urgent, cold }) => + config.queue.lanes.enabled ? laneFor({ fromSitemap, renderInterval, urgent, cold }) : 0; + /** * The batch form, for the fan-out writers (a target's device variants, `Target.revalidate`, * sitemap ingest, a reconcile repair pass). Writes every row, then lowers the floor ONCE with @@ -259,18 +352,27 @@ export const writeSchedule = async (cacheKey, { nextRenderTime, fromSitemap } = * earlier rows applied — the same semantics as before, and deletes/puts here are idempotent. */ export const writeSchedules = async (rows = []) => { - let lowest = Number.POSITIVE_INFINITY; - for (const { cacheKey, nextRenderTime, fromSitemap } of rows) { + // PER LANE, not one minimum for the batch. A fan-out can legitimately span lanes — a device pair + // shares a lane, but `Target.revalidate` over a sitemap walk and a reconcile repair both write + // mixed provenance — and one global minimum would lower whichever lane happened to own the + // earliest row while leaving the others' floors above rows they now have to find. That is the + // stranding this funnel exists to prevent, so the minimum is tracked per lane. + const lowestByLane = new Map(); + for (const { cacheKey, nextRenderTime, fromSitemap, renderInterval, urgent, cold } of rows) { if (fromSitemap === undefined) { throw new Error(`writeSchedules(${cacheKey}) needs an explicit fromSitemap — put replaces the record`); } - await scheduleTable().put(cacheKey, { nextRenderTime, fromSitemap }); + const lane = laneToWrite({ fromSitemap, renderInterval, urgent, cold }); + const stored = encodeDueAt(nextRenderTime, lane); + await scheduleTable().put(cacheKey, { nextRenderTime: stored, fromSitemap }); // Same trap as `lowerFloorFor`, and WORSE here: with a bare `Number` a single null row anywhere // in the batch becomes 0, wins the minimum, and unbounds the floor for the whole fan-out. - const at = numberOf(nextRenderTime); - if (Number.isFinite(at) && at < lowest) lowest = at; + const at = numberOf(stored); + if (Number.isFinite(at) && at < (lowestByLane.get(lane) ?? Number.POSITIVE_INFINITY)) { + lowestByLane.set(lane, at); + } } - if (lowest !== Number.POSITIVE_INFINITY) lowerFloorFor(lowest); + for (const [lane, lowest] of lowestByLane) lowerFloorFor(lowest, lane); }; /** Drop a schedule row. Lowers nothing, releases nothing — see the module comment. */ @@ -283,8 +385,17 @@ export const deleteSchedule = async (cacheKey) => { * residency-pinned, so a point read of a key this node does not own takes Harper's replication * fetch, which has NO TIMEOUT and can hang the caller forever. */ -export const getScheduleRow = (cacheKey, select) => - scheduleTable().get({ id: cacheKey, select }, { replicateFrom: false }); +export const getScheduleRow = async (cacheKey, select) => { + const row = await scheduleTable().get({ id: cacheKey, select }, { replicateFrom: false }); + // DECODED ON THE WAY OUT, ALWAYS — including while `queue.lanes.enabled` is false. Every reader + // in the plugin treats `nextRenderTime` as a timestamp (the invalidation accelerator does + // `nextRenderTime - interval` arithmetic on it; the console renders it as a date), and an encoded + // value used as one is a date ~139 years per lane in the future: plausible-looking, and wrong. + // Decoding unconditionally is also what makes disabling lanes a survivable rollback rather than a + // corpus-wide stranding — rows written while it was on still read as their real due times. + if (!row || row.nextRenderTime === undefined || row.nextRenderTime === null) return row; + return { ...row, nextRenderTime: dueAtOf(row.nextRenderTime), lane: laneOf(row.nextRenderTime) }; +}; // ---- the claim pass ------------------------------------------------------------------------- @@ -307,9 +418,28 @@ export const runClaimPass = async ({ leaseTimeMs, floorEnabled = true, floorRule = 'first-due-observed', + // ---- lane awareness. Every one of these defaults to the pre-lane behaviour, so an unlaned + // call is byte-for-byte the function that existed before `util/renderLane.js`, and every + // existing trace through it still describes it exactly. + // + // `lane` is passed through to `searchSchedules` (which builds the lane-scoped seek bound) and + // used to recognize SPILL: the seek has one condition and no upper bound — a two-sided range + // costs 1,128-2,977 ms on this index, so the upper bound stays in application code — which + // means a sparse lane's window runs on into the next lane's rows. Those are recognized and + // dropped, not granted. + // + // `floors` is whatever owns this lane's watermark. It defaults to `leases` because the single + // global floor lives in the lease buffer; a laned pass hands in an adapter over + // `util/laneFloor.js` instead. The three methods used are the same either way. + lane = 0, + floors = null, + decode = null, } = {}) => { + const watermark = floors ?? leases; + // Identity by default: an unencoded value IS its due time, in lane 0. + const readRow = decode ?? ((raw) => ({ lane: 0, dueAt: numberOf(raw) })); const nowMinute = minuteOf(nowMs); - const floorFrom = floorEnabled ? leases.readFloorMinute(nowMinute, guard) : 0; + const floorFrom = floorEnabled ? watermark.readFloorMinute(nowMinute, guard) : 0; // A leased row keeps its overdue position in the index now, so the pass must read PAST the // in-flight pile to find grantable rows: grantLimit to cover the pile's own head, the pile @@ -324,7 +454,7 @@ export const runClaimPass = async ({ // iterator leaves its read transaction unreleased (util/reconcile.js:60-64). The app-side cut // at "past now" is applied to the drained array below, not by walking away from the cursor. const rows = []; - for await (const row of searchSchedules({ floorMinute: floorFrom, limit: scanLimit })) rows.push(row); + for await (const row of searchSchedules({ lane, floorMinute: floorFrom, limit: scanLimit })) rows.push(row); const jobs = []; let sawDue = false; @@ -343,17 +473,27 @@ export const runClaimPass = async ({ let leaseRefused = false; let skippedLeased = 0; let nonFinite = 0; + let spilled = 0; for (const row of rows) { // `numberOf` because `Number(null)` is 0, which reads as "due since 1970" and would make an // absent due time the oldest due row in the corpus — pinning the floor at the epoch and naming // the wrong key as the row holding it. A missing due time is skipped and counted, not coerced. - const at = numberOf(row.nextRenderTime); + const { lane: rowLane, dueAt: at } = readRow(row.nextRenderTime); if (!Number.isFinite(at)) { nonFinite++; continue; } + // SPILL. Rows arrive ascending by the ENCODED value, so every row of this lane precedes every + // row of the next one — the first foreign row means this lane is exhausted. Counted separately + // from `nonFinite` because it is not a defect: it is the read cost of keeping the query to one + // condition, and it is the number that says whether a lane's scan limit is being wasted. + if (rowLane !== lane) { + spilled++; + continue; + } + if (at > nowMs) { earliestNotYetDueMinute = minuteOf(at); break; @@ -396,27 +536,29 @@ export const runClaimPass = async ({ if (floorEnabled) { // CAS against the value this pass started from, and ABANDON on conflict — a conflict means // a funnel write lowered the floor for a row this pass never saw. The next pass re-advances. - floorAdvanced = leases.advanceFloor(floorFrom, floorTo); + floorAdvanced = watermark.advanceFloor(floorFrom, floorTo); } else { // The kill switch forces the floor to 0 and changes nothing else, so re-enabling it starts // from a full seek rather than from a value that has been going stale. - leases.resetFloor(); + watermark.resetFloor(); } - leases.recordPassOutcome({ sawDue, earliestNotYetDueMinute }); + watermark.recordPassOutcome?.({ sawDue, earliestNotYetDueMinute }); // How long the SAME row has been holding the floor, node-wide. Recorded here rather than derived // by a caller because this is the only place that knows which row the floor rule actually picked, // and it is what both the wedged-row warning and the unpin escape hatch key off. `null` clears it, // so a pass that finds nothing due does not leave a stale pin ageing forever. - const floorPinnedForMs = leases.notePinnedBy(floorHeldBy); + const floorPinnedForMs = watermark.notePinnedBy(floorHeldBy); return { + lane, jobs, sawDue, granted: jobs.length, skippedLeased, nonFinite, + spilled, earliestNotYetDueMinute, floorFrom, floorTo, @@ -484,7 +626,7 @@ let lastFloorHeldByAt = 0; * future minute, so it changes nothing. A failure here is logged and swallowed: the claim must not 500 * because a repair could not be written, and the next pass simply tries again. */ -const maybeUnpinFloor = async (pass) => { +const maybeUnpinFloor = async (pass, lane = null) => { const unpinAfter = config.queue.claimFloor.unpinAfter; if (!(unpinAfter > 0)) return null; if (!config.queue.claimFloor.enabled) return null; @@ -520,7 +662,11 @@ const maybeUnpinFloor = async (pass) => { // Clear the pin so the promoted row starts its own clock from this pass rather than inheriting // this one's age — without it the next pass would qualify immediately and unpin a healthy row. - leaseTable().notePinnedBy(null); + // Cleared on whichever watermark the pin was recorded against: a laned pass pins per lane, and + // clearing the global one instead would leave the lane's pin ageing forever while the hatch + // re-fired on a healthy row every interval. + if (lane === null) leaseTable().notePinnedBy(null); + else laneFloors().notePinnedBy(lane, null); logger.warn( `[prerender] ${cacheKey} held the claim queue's floor for ${Math.round(pass.floorPinnedForMs / 60_000)} minute(s) ` + @@ -533,9 +679,9 @@ const maybeUnpinFloor = async (pass) => { }; /** `runClaimPass` bound to the live table and config. Called by `RenderQueue.claim`. */ -export const claimSchedules = async ({ grantLimit } = {}) => { +const claimOneLane = async ({ grantLimit, lane = 0, laned = false }) => { const pass = await runClaimPass({ - searchSchedules: ({ floorMinute, limit }) => + searchSchedules: ({ lane: seekLane, floorMinute, limit }) => scheduleTable().search( { // EXACTLY ONE CONDITION, and it stays present even at floorMinute 0 (`>= 0` is the @@ -555,7 +701,17 @@ export const claimSchedules = async ({ grantLimit } = {}) => { // of its time in. The `<= now` half stays in application code, where it is free. // (On a PRIMARY key a two-sided range collapses to a filtered intersection — // 289–1490 ms — which is why the shape of this query is worth a comment at all.) - conditions: [{ attribute: 'nextRenderTime', comparator: 'greater_than_equal', value: floorMinute * MINUTE }], + // LANE-SCOPED, STILL ONE CONDITION. The lane's slice starts at `lane * LANE_STRIDE`, so + // its seek bound is that plus the lane's own watermark. There is deliberately no upper + // bound: adding one is the two-sided range measured above, so a sparse lane's window + // runs on into the next lane's rows and `runClaimPass` drops them as SPILL instead. + conditions: [ + { + attribute: 'nextRenderTime', + comparator: 'greater_than_equal', + value: seekLane * LANE_STRIDE + floorMinute * MINUTE, + }, + ], sort: { attribute: 'nextRenderTime' }, // ARRAY select. A string `select` returns the bare scalar rather than a record — // the trap that has caused two silent bugs in this package already. @@ -571,20 +727,221 @@ export const claimSchedules = async ({ grantLimit } = {}) => { scanCap: Math.max(1, config.queue.claimScanCap | 0), leaseTimeMs: config.queue.jobLeaseTime, floorEnabled: config.queue.claimFloor.enabled, + lane, + // The un-laned path keeps the GLOBAL floor and the identity decode, so it is the function that + // existed before lanes, byte for byte. The laned path swaps in that lane's watermark and the + // real decoder — and note the decode is only swapped for the CLAIM; `getScheduleRow` decodes + // unconditionally, because a reader that gets this wrong is silently 139 years out. + floors: laned ? floorsForLane(lane) : null, + decode: laned ? (raw) => ({ lane: laneOf(raw), dueAt: dueAtOf(raw) }) : null, }); // Whatever this pass saw, including `null` for "nothing is due": a stale key here would name an // innocent URL as the thing pinning the queue. - lastFloorHeldBy = pass.floorHeldBy; - lastFloorHeldByAt = Date.now(); + // + // NOT SET BY A LANE PASS. A laned claim runs this function once per lane, and the last lane is + // usually `cold` with nothing due — so recording here would overwrite a genuinely pinned lane's + // key with the empty lane's `null` on every claim, and `floorState` (and therefore the console's + // "Claim floor lag") would report that nothing is pinned while a lane was wedged. The laned path + // records once, from `mergeLanePasses`, which is the only place that knows which lane's pin is the + // one worth naming. + if (!laned) { + lastFloorHeldBy = pass.floorHeldBy; + lastFloorHeldByAt = Date.now(); + } // AFTER the pass, never inside it: `runClaimPass` takes all its I/O as arguments precisely so the // floor algebra has no database in it, and a write issued mid-pass would also be a write with the // scan cursor still open (see the drain note above). - const floorUnpinned = await maybeUnpinFloor(pass); + const floorUnpinned = await maybeUnpinFloor(pass, laned ? lane : null); return floorUnpinned ? { ...pass, floorUnpinned } : pass; }; +/** + * ONE CLAIM, ACROSS LANES. + * + * With `queue.lanes.enabled` false this is one call to `claimOneLane` and nothing else — the same + * single pass, the same global floor, the same numbers. Everything below only happens when lanes + * are on. + * + * ── WHY A SEEK PER LANE RATHER THAN ONE SCAN ──────────────────────────────────────────────────── + * + * Because the whole problem is that one scan can only start in one place. The claim window is + * anchored at the oldest due time in the corpus, and under a backlog every row in it is ancient — + * so a homepage two of its own cadences late is nowhere near the window, and no amount of + * re-ranking inside it can find a row it never reads. A lane is a disjoint slice of the same index + * with a watermark of its own, so its head is ITS oldest row, not the corpus's. + * + * Measured (#80): three lanes interleaved in one index, each with its own watermark, 0.29-0.32 ms + * per lane. The same lane without a watermark, 3.46 ms. Interleaving is free; the watermark is the + * win. That is what makes N seeks cheaper than the one degraded seek this replaces. + * + * ── HOW THE BATCH IS DIVIDED ──────────────────────────────────────────────────────────────────── + * + * Lane order, with `urgent` capped by its drain share and the protected classes floored — see + * `createLaneBudget` for why floors rather than fixed shares, and why strict priority is not an + * option at any capacity level. The TOP-UP sweep at the end is what makes a floor a minimum rather + * than an entitlement: a class with fewer due rows than its floor releases the difference back to + * lane order instead of leaving that slice of the batch unspent. + * + * The sweep is skipped whenever the first pass spent the batch, which is the backlogged case — so + * the extra seeks happen exactly when the queue is caught up and they are cheapest. + */ +export const claimSchedules = async ({ grantLimit } = {}) => { + if (!config.queue.lanes.enabled) return claimOneLane({ grantLimit }); + + const { urgentMaxShare, minShare } = config.queue.lanes; + const budget = createLaneBudget({ grantLimit, urgentMaxShare, minShare }); + const order = Array.from({ length: laneCount() }, (_, lane) => lane); + const passes = []; + + for (const lane of order) { + const allowance = budget.allowanceFor(lane); + if (allowance <= 0) continue; + const pass = await claimOneLane({ grantLimit: allowance, lane, laned: true }); + budget.record(lane, pass.granted); + passes.push(pass); + } + + // Release unclaimed reservations and sweep again, in lane order. + if (budget.remaining > 0) { + budget.topUp(); + for (const lane of order) { + const allowance = budget.allowanceFor(lane); + if (allowance <= 0) continue; + const pass = await claimOneLane({ grantLimit: allowance, lane, laned: true }); + budget.record(lane, pass.granted); + passes.push(pass); + } + } + + return mergeLanePasses(passes, grantLimit); +}; + +/** + * Fold the per-lane passes into the one shape `RenderQueue.claim` consumes. + * + * The interesting choice is WHICH LANE'S FLOOR gets reported as "the" floor, because the console + * shows one number and the wedged-row warning names one URL. It reports the lane that has been + * PINNED LONGEST, not the lane with the oldest floor: an old floor on a lane that is draining is + * normal (that is what a backlog looks like), whereas a floor that has not moved is the one failure + * this reporting exists to catch. Ties go to the older floor. Per-lane detail rides along in + * `lanes` so nothing is hidden behind that choice. + */ +const mergeLanePasses = (passes, grantLimit) => { + if (passes.length === 0) { + return { + jobs: [], + sawDue: false, + granted: 0, + skippedLeased: 0, + nonFinite: 0, + spilled: 0, + earliestNotYetDueMinute: 0, + floorFrom: 0, + floorTo: 0, + floorHeldBy: null, + floorHeldByRow: null, + floorPinnedForMs: 0, + floorAdvanced: false, + scanned: 0, + scanLimit: 0, + scanTruncated: false, + leaseRefused: false, + occupancy: leaseTable().occupancy(), + lanes: [], + grantLimit, + }; + } + + // A pass with no due row has no pin to report, so it must not win the "longest pinned" contest + // with a floorPinnedForMs of 0 against a lane that genuinely has one. + const pinned = passes.filter((p) => p.floorHeldBy); + const worst = + pinned.length === 0 + ? passes[0] + : pinned.reduce((a, b) => + b.floorPinnedForMs > a.floorPinnedForMs || + (b.floorPinnedForMs === a.floorPinnedForMs && b.floorTo < a.floorTo) + ? b + : a + ); + + const sum = (key) => passes.reduce((total, p) => total + (p[key] || 0), 0); + + lastFloorHeldBy = worst.floorHeldBy ?? null; + lastFloorHeldByAt = Date.now(); + + return { + jobs: passes.flatMap((p) => p.jobs), + sawDue: passes.some((p) => p.sawDue), + granted: sum('granted'), + skippedLeased: sum('skippedLeased'), + nonFinite: sum('nonFinite'), + spilled: sum('spilled'), + // The soonest anything is due anywhere, ignoring lanes that saw nothing at all (0 is "unknown" + // in this field, not "the epoch"), because it feeds the node's empty-vs-queued derivation. + earliestNotYetDueMinute: Math.min(...passes.map((p) => p.earliestNotYetDueMinute || Infinity)) || 0, + floorFrom: worst.floorFrom, + floorTo: worst.floorTo, + floorHeldBy: worst.floorHeldBy, + floorHeldByRow: worst.floorHeldByRow, + floorPinnedForMs: worst.floorPinnedForMs, + floorLane: worst.lane, + floorAdvanced: passes.some((p) => p.floorAdvanced), + scanned: sum('scanned'), + scanLimit: Math.max(...passes.map((p) => p.scanLimit)), + scanTruncated: passes.some((p) => p.scanTruncated), + leaseRefused: passes.some((p) => p.leaseRefused), + occupancy: leaseTable().occupancy(), + floorUnpinned: passes.find((p) => p.floorUnpinned)?.floorUnpinned, + lanes: passes.map((p) => ({ + lane: p.lane, + label: laneLabel(p.lane), + class: classOfLane(p.lane), + granted: p.granted, + sawDue: p.sawDue, + scanned: p.scanned, + spilled: p.spilled, + skippedLeased: p.skippedLeased, + floorTo: p.floorTo, + floorHeldBy: p.floorHeldBy, + floorPinnedForMs: p.floorPinnedForMs, + })), + grantLimit, + }; +}; + +/** + * Run one bounded lane-restamp pass. See `util/laneRestamp.js` for what this is for and why it is + * needed at all given that lanes are otherwise derived at write time. + * + * The I/O lives HERE because this module is the only file allowed to touch `RenderSchedule` — the + * same rule that keeps the floor lowering inseparable from the due-time write, enforced by + * `test/queueFunnel.test.js`. `laneRestamp.js` owns the decision; this owns the table. + */ +export const restampLanes = async ({ limit = 5000, force = false } = {}) => { + const guard = restampGuard({ force }); + if (!guard.allowed) return { error: guard.reason }; + + const result = await restampPass({ + limit, + searchUnstamped: ({ limit: cap }) => scheduleTable().search(unstampedQuery(cap), { replicateFrom: false }), + // Through `writeSchedules`, not a raw put, so each batch lowers its lane's watermark with the + // batch minimum. A raw write here would file every restamped row BELOW the lane floor the very + // first pass then establishes — the terminal render gap, applied to the whole corpus at once. + writeRow: (rows) => writeSchedules(rows), + }); + + if (result.restamped) { + logger.info( + `[prerender] lane restamp: moved ${result.restamped} of ${result.examined} row(s) ` + + `(${result.lanes.map((l) => `${l.label}=${l.rows}`).join(' ')})${result.done ? ' — nothing left to move' : ''}` + ); + } + return result; +}; + // ---- lease lifecycle exposed to the result path --------------------------------------------- export const releaseLease = (cacheKey) => leaseTable().release(cacheKey); diff --git a/packages/plugin/test/queueFunnel.test.js b/packages/plugin/test/queueFunnel.test.js index 6769de0..c768936 100644 --- a/packages/plugin/test/queueFunnel.test.js +++ b/packages/plugin/test/queueFunnel.test.js @@ -115,8 +115,15 @@ test('the funnel owns the claim floor: nothing else touches the lease table’s // `resetFloorNow`, and the console reads through `floorState`, so neither needs these.) const primitives = /\b(?:advanceFloor|lowerFloorTo|resetFloor)\s*\(/; + // `util/renderLease.js` and `util/laneFloor.js` DEFINE these primitives — one for the single + // global floor, one for the per-lane watermarks — so they are the two files the rule cannot + // apply to. Both are dependency-free data structures over a plain ArrayBuffer for exactly this + // reason: the CAS rules live somewhere testable, and the decision about when to invoke them + // lives in the funnel and nowhere else. + const DEFINERS = new Set(['util/renderLease.js', 'util/laneFloor.js']); + for (const [path, source] of sources) { - if (path === FUNNEL || path === 'util/renderLease.js') continue; + if (path === FUNNEL || DEFINERS.has(path)) continue; assert.equal( primitives.test(source), false, diff --git a/packages/plugin/test/renderLane.test.js b/packages/plugin/test/renderLane.test.js new file mode 100644 index 0000000..08c0712 --- /dev/null +++ b/packages/plugin/test/renderLane.test.js @@ -0,0 +1,469 @@ +import { test, before, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; + +/** + * Render lanes — the encoding, the taxonomy, the fairness allocator, the per-lane watermarks, and + * the claim pass over all of it. + * + * What is pinned here, and why each one is a bug nothing else in this package would catch: + * + * - THE ENCODING ROUND-TRIPS EXACTLY, including at `Long` magnitudes. A due time recovered one + * millisecond wrong is a due time; recovered 139 years wrong it is a page that never renders. + * - AN ABSENT DUE TIME SURVIVES THE ENCODING. `Number(null)` is 0 and 0 is finite, so a bare + * coercion turns "no due time" into "due at the epoch", and a floor of 0 means NO FLOOR — one + * null row would put the whole claim scan back to seeking the absolute index minimum. + * - A LANE'S WATERMARK CANNOT STRAND ANOTHER LANE. This is the whole point: the failure the design + * replaces is one wedged row holding the scan position for every other route. + * - THE FLOOR RULE STILL HOLDS PER LANE. Same rule, same hazard — a floor advanced past a row the + * pass observed is a permanently unclaimable row, and it is silent. + * - SPILL IS DROPPED, NOT GRANTED. The lane seek has one condition and no upper bound (a two-sided + * range costs 1,128-2,977 ms on this index), so a sparse lane reads into the next lane's rows. + * Granting one would render a job under another lane's budget. + * - FLOORS ARE MINIMUMS, NOT ENTITLEMENTS. An unclaimed reservation has to come back, or a class + * with nothing due silently shrinks every batch. + * - `urgentMaxShare` IS A CAP THAT HOLDS EVEN WHEN NOTHING ELSE WANTS THE CAPACITY. It is the only + * structural bound stopping a bulk force-render from becoming a queue-wide outage. + */ + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const T0 = 1_700_000_400_000; // a whole minute +const minuteOf = (ms) => Math.floor(ms / MINUTE); + +let lane, floors, funnel, leaseMod, config; + +const sabs = new Map(); + +before(async () => { + globalThis.server = { hostname: 'test-node', nodes: [], config: { http: { port: 9926 } } }; + globalThis.logger = { debug() {}, info() {}, warn() {}, error() {} }; + globalThis.databases = { + coordination: { + SharedBuffer: { + primaryStore: { + getUserSharedBuffer: (key, buffer) => { + if (!sabs.has(key)) sabs.set(key, buffer); + return sabs.get(key); + }, + tryLock: () => true, + unlock() {}, + }, + }, + }, + render_schedule: { RenderSchedule: { put: async () => {}, delete: async () => {}, search: () => [] } }, + }; + + ({ config } = await import('../src/config.js')); + lane = await import('../src/util/renderLane.js'); + floors = await import('../src/util/laneFloor.js'); + funnel = await import('../src/util/renderSchedule.js'); + leaseMod = await import('../src/util/renderLease.js'); +}); + +beforeEach(() => { + // The defaults this file reasons about: two cuts -> three bands per banded class, so + // urgent(0) submitted(1,2,3) discovered(4,5,6) cold(7). + config.queue.lanes.ttlBands = [HOUR, 12 * HOUR]; +}); + +// ---- the encoding ------------------------------------------------------------------------------ + +test('encode/decode round-trips every lane exactly', () => { + for (const l of [0, 1, 3, 7, 12]) { + const encoded = lane.encodeDueAt(T0, l); + assert.equal(lane.laneOf(encoded), l, `lane ${l}`); + assert.equal(lane.dueAtOf(encoded), T0, `dueAt in lane ${l}`); + } +}); + +test('lane 0 is the IDENTITY, which is what makes every pre-existing row a valid lane-0 row', () => { + assert.equal(lane.encodeDueAt(T0, 0), T0); + assert.equal(lane.laneOf(T0), 0); + assert.equal(lane.dueAtOf(T0), T0); +}); + +test('lower encoded value is claimed first, so lane order IS priority order', () => { + // The property the whole design rests on: every row of lane N sorts before every row of N+1, + // however overdue the later lane's rows are. A three-day-overdue product page cannot outrank a + // homepage in a faster lane. + const threeDaysOverdue = lane.encodeDueAt(T0 - 3 * 24 * HOUR, 3); + const onTimeFastLane = lane.encodeDueAt(T0, 1); + assert.ok(onTimeFastLane < threeDaysOverdue); +}); + +test('an ABSENT due time survives encoding unchanged — 0 is finite and a floor of 0 means NO floor', () => { + assert.equal(lane.encodeDueAt(null, 3), null); + assert.equal(lane.encodeDueAt(undefined, 3), undefined); + assert.equal(lane.dueAtOf(null), null); + assert.equal(lane.dueAtOf(undefined), undefined); + // ...while a REAL 0 still encodes. It is the documented `nextRenderTime = 1` shape and must work. + assert.equal(lane.dueAtOf(lane.encodeDueAt(1, 2)), 1); + assert.equal(lane.laneOf(lane.encodeDueAt(1, 2)), 2); +}); + +test('a BigInt from a Long column decodes rather than throwing', () => { + const encoded = BigInt(lane.encodeDueAt(T0, 4)); + assert.equal(lane.laneOf(encoded), 4); + assert.equal(lane.dueAtOf(encoded), T0); +}); + +test('the stride leaves room for every lane the taxonomy can produce, inside a 52-bit Long', () => { + assert.ok(lane.laneCount() < lane.MAX_LANES, `${lane.laneCount()} lanes must fit in ${lane.MAX_LANES}`); + const highest = lane.encodeDueAt(T0, lane.laneCount() - 1); + assert.ok(Number.isSafeInteger(highest), 'the highest encodable due time must stay a safe integer'); +}); + +// ---- the taxonomy ----------------------------------------------------------------------------- + +test('TTL bands split a class so EDF inside a lane approximates relative lateness', () => { + const submitted = (interval) => lane.laneFor({ fromSitemap: true, renderInterval: interval }); + // The three cadences this deployment actually runs must land in three different lanes, or the + // homepage goes on losing to product pages that are older in absolute terms. + assert.notEqual(submitted(HOUR), submitted(6 * HOUR)); + assert.notEqual(submitted(6 * HOUR), submitted(48 * HOUR)); + assert.ok(submitted(HOUR) < submitted(6 * HOUR)); + assert.ok(submitted(6 * HOUR) < submitted(48 * HOUR)); +}); + +test('submitted outranks discovered at the same cadence, and both outrank cold', () => { + const s = lane.laneFor({ fromSitemap: true, renderInterval: HOUR }); + const d = lane.laneFor({ fromSitemap: false, renderInterval: HOUR }); + const c = lane.laneFor({ fromSitemap: true, renderInterval: HOUR, cold: true }); + assert.ok(s < d, 'submitted before discovered'); + assert.ok(d < c, 'discovered before cold'); + assert.equal(lane.laneFor({ fromSitemap: false, renderInterval: 48 * HOUR, urgent: true }), 0); +}); + +test('an ABSENT cadence takes the SLOWEST band, never the fastest', () => { + // The safe direction: an absent interval is an absent claim to urgency. Defaulting to band 0 + // would let any row that lost its cadence jump ahead of the homepage. + const slowest = lane.laneFor({ fromSitemap: true, renderInterval: 999 * HOUR }); + for (const missing of [undefined, null, 0, -1, Number.NaN, 'x']) { + assert.equal(lane.laneFor({ fromSitemap: true, renderInterval: missing }), slowest, `interval=${missing}`); + } +}); + +test('an unsorted or duplicated band list still produces a monotonic order', () => { + // The band index IS part of the stored key, so an unsorted list would file a shorter interval in + // a later lane and quietly invert the ordering it exists to create. + config.queue.lanes.ttlBands = [12 * HOUR, HOUR, HOUR, -5, Number.NaN]; + const submitted = (i) => lane.laneFor({ fromSitemap: true, renderInterval: i }); + assert.ok(submitted(HOUR) < submitted(6 * HOUR)); + assert.ok(submitted(6 * HOUR) < submitted(48 * HOUR)); +}); + +test('classOfLane and laneLabel name every lane the taxonomy produces', () => { + const seen = new Set(); + for (let l = 0; l < lane.laneCount(); l++) seen.add(lane.classOfLane(l)); + assert.deepEqual([...seen].sort(), ['cold', 'discovered', 'submitted', 'urgent']); + assert.equal(lane.laneLabel(0), 'urgent'); + assert.equal(lane.laneLabel(lane.laneCount() - 1), 'cold'); + assert.equal(lane.laneLabel(1), 'submitted/b0'); +}); + +test('an empty band list degrades to provenance-only priority rather than breaking', () => { + config.queue.lanes.ttlBands = []; + assert.equal(lane.bandCount(), 1); + assert.equal(lane.laneCount(), 4); + assert.equal(lane.laneFor({ fromSitemap: true, renderInterval: HOUR }), 1); + assert.equal(lane.laneFor({ fromSitemap: true, renderInterval: 48 * HOUR }), 1); + assert.equal(lane.laneFor({ fromSitemap: false, renderInterval: HOUR }), 2); +}); + +// ---- the fairness allocator ------------------------------------------------------------------- + +const budgetWith = (opts) => lane.createLaneBudget({ grantLimit: 20, ...opts }); + +test('urgentMaxShare caps the urgent lane even when nothing else wants the capacity', () => { + const budget = budgetWith({ urgentMaxShare: 0.2 }); + assert.equal(budget.allowanceFor(0), 4, '20% of 20'); + budget.record(0, 4); + assert.equal(budget.allowanceFor(0), 0, 'and it does not refill within the pass'); + // The rest of the batch is still available to the lanes below — that is the structural bound. + assert.equal(budget.remaining, 16); +}); + +test('a share too small to floor to a whole job still admits one', () => { + // A cap of zero would make an operator's force-render silently never run, with nothing to say why. + assert.equal(budgetWith({ grantLimit: 3, urgentMaxShare: 0.2 }).allowanceFor(0), 1); + // ...and an explicit 0 really does disable the lane. + assert.equal(budgetWith({ urgentMaxShare: 0 }).allowanceFor(0), 0); +}); + +test('a protected class floor is held back from the lanes above it', () => { + const budget = budgetWith({ urgentMaxShare: 0, minShare: { discovered: 0.1, cold: 0.05 } }); + // discovered reserves 2, cold reserves 1, so the submitted lanes may take at most 17. + const submitted = lane.laneFor({ fromSitemap: true, renderInterval: HOUR }); + assert.equal(budget.allowanceFor(submitted), 17); + budget.record(submitted, 17); + // ...and the reservations are then exactly what is left, for the classes they were held for. + const discovered = lane.laneFor({ fromSitemap: false, renderInterval: HOUR }); + assert.equal(budget.allowanceFor(discovered), 2); +}); + +test('a class is never held back by its OWN floor', () => { + // Its siblings' reservations must not be able to starve it on the mechanism meant to protect it. + const budget = budgetWith({ urgentMaxShare: 0, minShare: { discovered: 1 } }); + const discovered = lane.laneFor({ fromSitemap: false, renderInterval: HOUR }); + assert.equal(budget.allowanceFor(discovered), 20); +}); + +test('FLOORS ARE MINIMUMS: an unclaimed reservation comes back to lane order', () => { + const budget = budgetWith({ urgentMaxShare: 0, minShare: { discovered: 0.5 } }); + const submitted = lane.laneFor({ fromSitemap: true, renderInterval: HOUR }); + assert.equal(budget.allowanceFor(submitted), 10, 'half the batch is reserved'); + budget.record(submitted, 10); + // Discovery had nothing due. Without the top-up, half of every batch would go unspent forever. + budget.topUp(); + assert.equal(budget.allowanceFor(submitted), 10); +}); + +test('the top-up does NOT release the urgent cap — a cap is not a reservation', () => { + const budget = budgetWith({ urgentMaxShare: 0.2, minShare: {} }); + budget.record(0, 4); + budget.topUp(); + assert.equal(budget.allowanceFor(0), 0, 'still capped after the top-up'); +}); + +// ---- the per-lane watermarks ------------------------------------------------------------------ + +const laneFloorTable = (lanes = 8, now = T0) => + floors.createLaneFloors({ buffer: new ArrayBuffer(floors.laneFloorBufferBytes(lanes)), lanes, now: () => now }); + +test('A LANE WATERMARK CANNOT STRAND ANOTHER LANE — the failure the design replaces', () => { + const table = laneFloorTable(); + // Lane 3 is wedged three days back; lane 1 has caught up to now. + table.advanceFloor(3, 0, minuteOf(T0 - 3 * 24 * HOUR)); + table.advanceFloor(1, 0, minuteOf(T0)); + assert.equal(table.readFloorMinute(1, minuteOf(T0), 0), minuteOf(T0)); + assert.equal(table.readFloorMinute(3, minuteOf(T0), 0), minuteOf(T0 - 3 * 24 * HOUR)); +}); + +test('the guard band holds a floor behind now, but never lifts an unbounded one', () => { + const table = laneFloorTable(); + table.advanceFloor(1, 0, minuteOf(T0)); + assert.equal(table.readFloorMinute(1, minuteOf(T0), 5), minuteOf(T0) - 5, 'clamped behind now'); + // A zero floor means "seek this lane's absolute minimum". Clamping it UP would silently turn + // re-derive-from-the-bottom into skip-everything-older-than-the-guard, stranding what a reset + // exists to recover. + assert.equal(table.readFloorMinute(2, minuteOf(T0), 5), 0); +}); + +test('lowerFloorTo is a CAS-min and advanceFloor abandons on conflict', () => { + const table = laneFloorTable(); + table.advanceFloor(1, 0, 500); + assert.equal(table.lowerFloorTo(1, 600), false, 'a later minute lowers nothing'); + assert.equal(table.lowerFloorTo(1, 400), true, 'an earlier one does'); + assert.equal(table.readFloorMinute(1, 10_000, 0), 400); + // A conflicting advance must ABANDON: a conflict means a write lowered the floor for a row the + // pass never saw, and re-advancing over it would strand that row. + assert.equal(table.advanceFloor(1, 500, 700), false); + assert.equal(table.readFloorMinute(1, 10_000, 0), 400); +}); + +test('a pin ages per lane, and clearing one does not clear another', () => { + let now = T0; + const table = floors.createLaneFloors({ + buffer: new ArrayBuffer(floors.laneFloorBufferBytes(8)), + lanes: 8, + now: () => now, + }); + assert.equal(table.notePinnedBy(1, 'a|desktop'), 0, 'a new pin starts at zero'); + assert.equal(table.notePinnedBy(3, 'b|desktop'), 0); + now = T0 + 10 * MINUTE; + assert.equal(table.notePinnedBy(1, 'a|desktop'), 10 * MINUTE, 'the same key keeps ageing'); + table.notePinnedBy(1, null); + assert.equal(table.notePinnedBy(1, 'a|desktop'), 0, 'cleared, so it restarts'); + assert.equal(table.notePinnedBy(3, 'b|desktop'), 10 * MINUTE, 'the other lane is untouched'); +}); + +test('a different key resets the pin clock — the pin is about a ROW, not a lane', () => { + let now = T0; + const table = floors.createLaneFloors({ + buffer: new ArrayBuffer(floors.laneFloorBufferBytes(4)), + lanes: 4, + now: () => now, + }); + table.notePinnedBy(1, 'a|desktop'); + now = T0 + 30 * MINUTE; + assert.equal(table.notePinnedBy(1, 'b|desktop'), 0, 'the promoted row starts its own clock'); +}); + +test('lanes are clamped to the buffer rather than indexing past it', () => { + const table = laneFloorTable(2); + assert.equal(table.laneCount, 2); + table.advanceFloor(99, 0, 400); + // Folded onto the last lane, not written out of bounds. + assert.equal(table.readFloorMinute(1, 10_000, 0), 400); +}); + +// ---- the claim pass, per lane ----------------------------------------------------------------- + +const harness = ({ rows, slots = 256, now = T0 }) => { + const leases = leaseMod.createLeaseTable({ + buffer: new ArrayBuffer(leaseMod.leaseBufferBytes(slots)), + slots, + now: () => now, + }); + const laneFloors = laneFloorTable(16, now); + const searchSchedules = ({ lane: seekLane, floorMinute, limit }) => + (async function* () { + const from = seekLane * lane.LANE_STRIDE + floorMinute * MINUTE; + const matching = rows + .filter((r) => Number(r.nextRenderTime) >= from) + .sort((a, b) => Number(a.nextRenderTime) - Number(b.nextRenderTime)) + .slice(0, limit); + for (const r of matching) yield { ...r }; + })(); + + const pass = (lane_, options = {}) => + funnel.runClaimPass({ + searchSchedules, + leases, + nowMs: now, + grantLimit: 20, + guardMinutes: 5, + scanCap: 1000, + leaseTimeMs: 10 * MINUTE, + floorEnabled: true, + lane: lane_, + floors: { + readFloorMinute: (nowMinute, guard) => laneFloors.readFloorMinute(lane_, nowMinute, guard), + advanceFloor: (from, to) => laneFloors.advanceFloor(lane_, from, to), + resetFloor: () => laneFloors.resetFloor(lane_), + notePinnedBy: (key) => laneFloors.notePinnedBy(lane_, key), + }, + decode: (raw) => ({ lane: lane.laneOf(raw), dueAt: lane.dueAtOf(raw) }), + ...options, + }); + + return { pass, leases, laneFloors }; +}; + +test("A DEEP BACKLOG IN A SLOW LANE DOES NOT HIDE THE FAST LANE'S ROWS — the whole point", () => { + // The production symptom, reproduced: 60 product rows three days overdue, and a homepage two + // hours overdue. Un-laned, the window is anchored at the oldest row and the homepage is never + // read at all. Laned, the fast lane's seek starts at ITS oldest row. + const rows = [ + ...Array.from({ length: 60 }, (_, i) => ({ + cacheKey: `pdp-${i}|desktop`, + nextRenderTime: lane.encodeDueAt(T0 - 3 * 24 * HOUR + i * MINUTE, 3), + fromSitemap: true, + })), + { cacheKey: 'home|desktop', nextRenderTime: lane.encodeDueAt(T0 - 2 * HOUR, 1), fromSitemap: true }, + ]; + const { pass } = harness({ rows }); + return pass(1, { grantLimit: 5 }).then((result) => { + assert.deepEqual( + result.jobs.map((j) => j.cacheKey), + ['home|desktop'], + 'the fast lane grants the homepage regardless of how deep the slow lane is' + ); + }); +}); + +test('SPILL from the next lane is dropped, never granted', async () => { + // Lane 1 is empty, so its one-sided seek reads straight into lane 2's rows. Granting one would + // render a job under lane 1's budget and lower lane 1's watermark for a row it does not own. + const rows = [ + { cacheKey: 'a|desktop', nextRenderTime: lane.encodeDueAt(T0 - HOUR, 2), fromSitemap: false }, + { cacheKey: 'b|desktop', nextRenderTime: lane.encodeDueAt(T0 - HOUR, 2), fromSitemap: false }, + ]; + const { pass, laneFloors } = harness({ rows }); + const result = await pass(1); + assert.equal(result.jobs.length, 0); + assert.equal(result.spilled, 2, 'counted, so a wasted scan window is visible'); + assert.equal(result.sawDue, false, 'an empty lane must not report the next lane as its own work'); + // And lane 2 still gets them. + const next = await pass(2); + assert.equal(next.jobs.length, 2); + assert.equal(laneFloors.readFloorMinute(2, minuteOf(T0), 0), minuteOf(T0 - HOUR)); +}); + +test('the floor rule holds per lane: the floor is the first DUE row that lane observed', async () => { + const rows = [ + { cacheKey: 'stuck|desktop', nextRenderTime: lane.encodeDueAt(T0 - 10 * HOUR, 2), fromSitemap: false }, + { cacheKey: 'ok|desktop', nextRenderTime: lane.encodeDueAt(T0 - HOUR, 2), fromSitemap: false }, + ]; + const { pass, leases } = harness({ rows }); + leases.grant('stuck|desktop', { dueMinute: minuteOf(T0 - 10 * HOUR), leaseExpiryMs: T0 + HOUR }); + + const result = await pass(2); + assert.deepEqual( + result.jobs.map((j) => j.cacheKey), + ['ok|desktop'] + ); + // The in-flight row is the first due row observed, so it holds the floor — a lease whose result + // may still be arriving must not have the floor advance past it. + assert.equal(result.floorTo, minuteOf(T0 - 10 * HOUR)); + assert.equal(result.floorHeldBy, 'stuck|desktop'); + assert.equal(result.skippedLeased, 1); +}); + +test('a not-yet-due row in a fast lane is still not granted', async () => { + const rows = [{ cacheKey: 'future|desktop', nextRenderTime: lane.encodeDueAt(T0 + HOUR, 1), fromSitemap: true }]; + const { pass } = harness({ rows }); + const result = await pass(1); + assert.equal(result.jobs.length, 0); + assert.equal(result.earliestNotYetDueMinute, minuteOf(T0 + HOUR)); +}); + +// ---- the restamp ------------------------------------------------------------------------------ + +test('the restamp moves a row into its lane WITHOUT changing its due time', async () => { + const { restampPass } = await import('../src/util/laneRestamp.js'); + const written = []; + const result = await restampPass({ + limit: 10, + searchUnstamped: () => + (async function* () { + yield { cacheKey: 'https://x/a|desktop', nextRenderTime: T0 - HOUR, fromSitemap: true }; + yield { cacheKey: 'https://x/b|desktop', nextRenderTime: T0 - 2 * HOUR, fromSitemap: false }; + yield { cacheKey: 'https://x/c|desktop', nextRenderTime: null, fromSitemap: true }; + })(), + writeRow: (rows) => written.push(rows), + }); + + assert.equal(result.examined, 3); + assert.equal(result.restamped, 2); + assert.equal(result.skipped, 1, 'a row with no usable due time is left alone, not given one'); + assert.equal(result.done, true, 'a pass that did not fill its window is finished'); + + // THE DUE TIMES ARE UNTOUCHED. The pass changes the order and nothing else, which is what makes + // it safe on a live node: no page renders sooner or later than it would have. + const all = written.flat(); + assert.deepEqual(all.map((r) => r.nextRenderTime).sort(), [T0 - 2 * HOUR, T0 - HOUR].sort()); + // ...and each batch is single-lane, so its watermark is lowered with that lane's own minimum. + for (const batch of written) { + const lanes = new Set( + batch.map((r) => lane.laneFor({ fromSitemap: r.fromSitemap, renderInterval: r.renderInterval })) + ); + assert.equal(lanes.size, 1); + } +}); + +test('a full window is NOT done, so the caller keeps going', async () => { + const { restampPass } = await import('../src/util/laneRestamp.js'); + const result = await restampPass({ + limit: 2, + searchUnstamped: () => + (async function* () { + yield { cacheKey: 'https://x/a|desktop', nextRenderTime: T0, fromSitemap: false }; + yield { cacheKey: 'https://x/b|desktop', nextRenderTime: T0, fromSitemap: false }; + })(), + writeRow: async () => {}, + }); + assert.equal(result.done, false); +}); + +test('the restamp refuses to run once lanes are enabled, because urgent and unstamped are one number', async () => { + const { restampGuard } = await import('../src/util/laneRestamp.js'); + config.queue.lanes.enabled = false; + assert.equal(restampGuard().allowed, true); + config.queue.lanes.enabled = true; + const guard = restampGuard(); + assert.equal(guard.allowed, false); + assert.match(guard.reason, /urgent/); + assert.equal(restampGuard({ force: true }).allowed, true); + config.queue.lanes.enabled = false; +});