diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 193b06a..e96eae9 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -18,10 +18,10 @@ name: Docker Image on: push: - branches: [ "main" ] + branches: [ "master" ] tags: [ "v*" ] pull_request: - branches: [ "main" ] + branches: [ "master" ] workflow_dispatch: # One run per ref: a tag push landing on top of a branch push should supersede diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70ff2b8..6387709 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,10 @@ on: push: branches: [ "master" ] pull_request: - branches: [ "master" ] + # Every base, not only master. A PR stacked on another feature branch is + # otherwise never checked at all: its first build is the merge into master, + # which is the one moment nobody wants to learn it does not compile. + branches: [ '**' ] workflow_dispatch: concurrency: @@ -74,7 +77,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 24 cache: npm cache-dependency-path: ui/package-lock.json diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..a45fd52 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/Cargo.lock b/Cargo.lock index ac31e20..2804455 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -460,6 +460,7 @@ dependencies = [ "chrono", "deadpool-postgres", "gate-core", + "getrandom 0.4.3", "jsonwebtoken", "mime_guess", "parking_lot", diff --git a/DESIGN_GATE_V2.md b/DESIGN_GATE_V2.md index b608c2d..e152e23 100644 --- a/DESIGN_GATE_V2.md +++ b/DESIGN_GATE_V2.md @@ -112,7 +112,7 @@ be unreadable by an older one, because that is what makes the store's `complete: "nodes": { "": { - "budgets": [ , ... ], // >= 1, and >= 1 of them unscoped + "budgets": [ , ... ], // >= 1; >= 1 unconditional and unscoped "cost": , // default: 1 "ingress": , // optional; absent = fed only by paths "egress": // optional; required on a terminal node @@ -381,9 +381,9 @@ function, and echoed in the declare response so a caller never has to reconstruc | interior queue | `gate.{app}.{graph}.{node}.in` | every non-ingress node has one | | egress queue | whatever the declaration names | Gate pushes, the app consumes | | **stage consumer group** | `gate.{app}.{graph}.{path}.{node}` | one per (path, node) — this is the whole group taxonomy | -| budget key (node) | `b:{app}:{graph}:{node}:{bid}` | namespace `gate` | -| budget key (scoped) | `b:{app}:{graph}:{node}:{bid}:{scopeValue}` | one row per value, TTL-reaped | -| budget key (shared) | `b:{app}:shared:{sharedKey}` | one row per app, across graphs | +| budget key (node) | `b:{enc(app)}:{localGraph(graph)}:{enc(node)}:{enc(bid)}` | namespace `gate` | +| budget key (scoped) | node key + `:{enc(scopeValue)}` | one row per value, TTL-reaped | +| budget key (shared) | `b:{enc(app)}:shared:{enc(sharedKey)}` | one row per app, across graphs | | breaker record | `brk:{app}:{graph}:{node}` | TTL = `retryAfterSeconds` | | spec store | `graph:{app}:{name}` | namespace `gate`, `Expiry::forever()` — unchanged from v1 | @@ -393,6 +393,11 @@ fail loudly, because the broker answers a group with no cursor with the queue's retained range — so an ETA built on a misspelling reports every message ever pushed as waiting for budget, plausibly, for ever. +`enc` percent-escapes `%` as `%25` and the structural `:` separator as `%3A`. Ordinary names +therefore keep their existing keys, while free-form budget ids, shared keys and scope values cannot +smuggle a separator. `localGraph` applies the same encoding and spells the legal graph name +`shared` as `%73hared`, because the unescaped word is the shared-budget namespace marker. + **One group per (path, node), not per node.** Two paths sharing an ingress node is **pub-sub**: each path's group gets **every** message, so the message traverses both paths. That is the documented, intended semantics and it composes with fan-out. It is also why the @@ -443,7 +448,7 @@ queen.queue(stage.source) // .subscription_from(runtime_start - INTERIOR_SEED_SKEW) .batch(stage.batch) // default 200 .partitions(1) // ONE source partition per claim — §6.4 - .concurrency(stage.concurrency) // default = max(4, source partitions) + .concurrency(stage.concurrency) // default derived from node-wide rate .auto_ack(false) // the relay settles inside its own txn .lease_seconds(30) // a WORK lease, not a pacing quantum .renew_lease(Duration::from_secs(10)) @@ -553,7 +558,8 @@ serving four read shapes. v2 runs **7 consumers, 1 reconcile loop, 1 history pru ### 5.1 One key, one counter, no window index ``` -key = b:{app}:{graph}:{node}:{bid}[:{scope}] (or b:{app}:shared:{sharedKey}) +key = b:{enc(app)}:{localGraph(graph)}:{enc(node)}:{enc(bid)}[:{enc(scope)}] + (or b:{enc(app)}:shared:{enc(sharedKey)}[:{enc(scope)}]) max = round(count_sub * share(path)) ttl = window_sub_seconds delta = cost @@ -896,8 +902,9 @@ The one piece of per-item provenance v2 keeps, unchanged in spirit from v1: "_gate": { "graph": "airbnb", "path": "prices", "hop": 2, "at": 1755763200000 } ``` -One reserved object, not four top-level keys, so it cannot collide with a `scopeBy` path or a -cost path. Stamped by the ingress push (HTTP front door) or by the first relay that handles an +One reserved object, not four top-level keys. The declaration path grammar explicitly rejects +`payload._gate` and anything below it, so neither `scopeBy` nor a cost path can read provenance as +producer data. Stamped by the ingress push (HTTP front door) or by the first relay that handles an unstamped message (which is how a user-owned ingress queue works — producers know nothing about Gate). Carried verbatim by every relay, rewritten per hop. It is **not signed and not verified**: it is trusted because it is written server-side and because writing to an interior @@ -1028,10 +1035,10 @@ Two broker calls: ``` depth = GET /api/v1/resources/queues/{node.source_queue}/depth?group={stage.group} -state = kv.batch([ getMany(NS, node.unscoped_budget_keys) ]) +state = kv.batch([ getMany(NS, node.node_wide_budget_keys) ]) ``` -Then, per budget `b`: +Then, per unconditional, unscoped budget `b`: ``` cap_p = round(count_sub(b) * share(path)) @@ -1050,6 +1057,12 @@ etaSeconds = max over b of seconds_b // the slowest budget binds boundBy = the b that produced it ``` +`whenOp` budgets are deliberately excluded from this queue-level bound. Depth +does not say which operations are waiting, so charging every queued item to a +selective counter would produce a late, false answer for non-matching work. The +ETA keeps the unconditional lower bound and names the unresolved selectors in +`assumes`. + `cap_p <= 0` (a share that rounds a path out of existence — refused at declare time, but a stored document from an older build can still carry it) answers `null`, never infinity. `null` rather than infinity because a product can render *"we cannot say"* and would render an @@ -1089,9 +1102,10 @@ should now be near `batch` rather than near 1. ### 10.3 The optional counters stream -`"counters": { "windowSeconds": 60 }` on the graph turns on **one** streams job per graph: a -tumbling-window aggregate over the egress queue producing `{ path, node, count, cost }` per -window, written to `gate.rollups`. This is opt-in, per graph, and off by default — the point +`"counters": { "windowSeconds": 60 }` on the graph turns on durable one-minute roll-ups. The +storage schema and history API are minute-keyed, so `60` is currently the only accepted value. +The runtime snapshots each replica's in-process stage counters and writes their deltas to +`gate.rollups`. This is opt-in, per graph, and off by default — the point of the architecture is that observability is a thing you switch on, not a thing that runs whether or not anyone is looking. It is the source for `avgCost`, `/api/flow`, `/api/rollups` and the console's charts. @@ -1131,12 +1145,14 @@ Rule names are asserted on in tests, so they are API. |---|---|---| | `nodes` | empty | `a graph with no nodes limits nothing.` | | `paths` | empty | `` a graph with no paths has no way in and no way out: declare at least one path naming the nodes a message visits, in order. `` | +| `counters-window` | `counters.windowSeconds != 60` | `` counters.windowSeconds is {n}, but Gate currently stores and serves fixed one-minute roll-ups. Set windowSeconds to 60, or omit counters to leave durable roll-ups off. `` | | `path-node` | a path names an undeclared node | `` path `{p}` visits `{n}`, which is not a declared node. Declared nodes are: {list}. `` | | `path-length` | a path has fewer than 1 element | `` path `{p}` is empty. `` | | `acyclic` | the union of all path edges has a cycle | `` these nodes form a cycle: {a} -> {b} -> {a}. An item would traverse it for ever, re-paying every budget on the way round. `` | | `path-entry` | a path's first node has no `ingress` | `` path `{p}` starts at `{n}`, which declares no ingress. Work cannot enter a node that has no queue to enter by: give `{n}` an `ingress`, or start the path at a node that has one. `` | | `path-terminal` | a path's last element contains a node with no `egress` | `` path `{p}` ends at `{n}`, which declares no egress. Work would be admitted and then have nowhere to go. Name the queue your consumers read: `"egress": "{app}.{graph}.out"`. `` | | `node-orphan` | a declared node appears in no path | `` node `{n}` is declared and no path visits it: it can never hold work. `` | +| `graph-workers` | the resolved stages total more than 4096 consumer workers | `` this graph resolves to {workers} consumer workers across {stages} stages; the maximum is 4096. Each worker allocates a task and a broker long-poll before the graph starts, so an unbounded value can exhaust a replica. Lower `nodes[].concurrency` or `GATE_STAGE_CONCURRENCY`, or split the topology into separate graphs. `` | | `fanout-branch` | a fan-out array has fewer than 2 elements, or nested arrays | `` path `{p}` hop {i}: a fan-out is a flat array of at least two node names. `` | | `fanout-terminal` | a fan-out is not the last hop of its path | `` path `{p}` fans out to {list} at hop {i}, which is not the last hop. After a fan-out the branches are separate streams; give each one its own path. `` | @@ -1145,7 +1161,8 @@ Rule names are asserted on in tests, so they are API. | rule | when | detail | |---|---|---| | `node-budget` | a node declares no budgets | `` node `{n}` declares no budget, so it limits nothing — it would admit everything straight through, which is a queue with extra steps. `` | -| `node-unscoped-budget` | every budget of a node has `scopeBy` | `` node `{n}` has only per-key budgets. It needs at least one budget on the node itself: it is what the ETA measures a rate against and what the breaker spends when a vendor says 429. `` | +| `node-unscoped-budget` | every budget of a node has `scopeBy` or `whenOp` | `` node `{n}` has no unconditional budget on the node itself. It needs at least one budget without scopeBy or whenOp: every item must meet that counter, so the ETA has a node-wide rate and the breaker can stop every operation when a vendor says 429. `` | +| `breaker-width` | a node compiles to more than 255 distinct unscoped counter keys | `` node `{n}` compiles to {k} distinct node-wide counters. A breaker must spend them and write its audit record atomically, but the broker accepts at most 256 operations in one call. Keep at most 255 distinct unscoped counters; budgets with the same sharedKey count once. `` | | `budget-count` | `count < 1` | `` budget `{b}` of node `{n}` has count {c}. A budget that cannot admit anything never will — no schedule refills it. `` | | `budget-window` | `timeMs < 100` | `` budget `{b}` of node `{n}` declares timeMs {t}. The floor is 100. `` | | `budget-window-floor` | `timeMs < 1000` | **WARNING, not a refusal** — see `window-sub-second` below. | @@ -1155,8 +1172,8 @@ Rule names are asserted on in tests, so they are API. | `cost-fits` | `cost.max > count_sub` for any budget | `` node `{n}`: an item may cost up to {max} and budget `{b}` admits {cs} per sub-window. An item that cannot fit a window can never be admitted — it parks the head of its partition for ever and never reaches a DLQ, because a lease that expires charges no retry. Raise the budget, lower cost.max, or lower subWindows. `` | | `cost-max` | `cost.max < cost.default` | `` node `{n}`: cost.max {m} is below cost.default {d}, so the default cost is itself inadmissible. `` | | `cost-integer` | a constant `cost` that is not an integer >= 1 | `` node `{n}`: cost must be a whole number of at least 1. The budget counter is an integer on this wire, so a fractional cost is not expressible — express the unit differently (count tenths, and multiply the budget by ten). `` | -| `cost-path` | a `path` that is not a dotted payload path | `` node `{n}`: cost.path `{p}` is not a payload path. Write it as `payload.field` or `payload.a.b`. `` | -| `scope-path` | `scopeBy` is not a dotted payload path | `` budget `{b}` of node `{n}`: scopeBy `{p}` is not a payload path. `` | +| `cost-path` | a `path` that is not a dotted payload path, or starts with reserved `payload._gate` | `` node `{n}`: cost.path `{p}` is not a payload path. Write it as `payload.field` or `payload.a.b`; `payload._gate` is reserved for Gate's routing stamp. `` | +| `scope-path` | `scopeBy` is not a dotted payload path, or starts with reserved `payload._gate` | `` budget `{b}` of node `{n}`: scopeBy `{p}` is not a payload path. Write it as `payload.field` or `payload.a.b`; `payload._gate` is reserved for Gate's routing stamp. `` | | `shared-conflict` | two budgets in this document share a `sharedKey` with different `count`/`timeMs`/`subWindows` | `` `{k}` is declared as {c1} per {t1}ms in node `{n1}` and {c2} per {t2}ms in node `{n2}`. They are one counter, so one of those declarations is a lie about what it enforces. Make them agree or give them different keys. `` | | `whenop-empty` | `whenOp: []` | `` budget `{b}` of node `{n}`: an empty whenOp matches nothing, so the budget charges nothing. Drop the field to take everything. `` | | `provenance` | `confidence: documented` with no `source` or no `asOf` | `` budget `{b}` of node `{n}` claims to be documented but names no {source/asOf}. A guess must never look like a measurement. `` | @@ -1260,7 +1277,9 @@ up to one old window to land, and the declare response says so. The rule is enforced for a **caller's** declare only, never for one applied from the store — enforcing it against a replica-local runtime is how a replica wedges on a legal -delete-and-redeclare at the same version. That asymmetry is v1's and it is kept verbatim. +delete-and-redeclare at the same version. A caller's declare compares both the local runtime and +the exact stored document, so reaching a replica before its reconcile pass cannot make an existing +graph look new and bypass the bump. That asymmetry is v1's and it is kept verbatim. ### 12.4 Drain and redeclare @@ -1404,7 +1423,7 @@ three hardcoded fields are fixed (§13.12). |---|---|---| | `PUT /v1/apps/:app/targets/:name` | R | Declares a one-node graph. Response keeps `resolved` (now: ingress queue, egress queue, stage groups, budget keys) and `warnings`. Still 502 on a store write failure — a 200 with a 15-second fuse is a lie. | | `PUT /v1/targets/:name` (flat) | R | Kept, **and the parity trap is fixed**: the flat form now pins `application` from the resolved default rather than letting a body declare into another team's namespace. | -| `PUT /v1/apps/:app/targets` (sync, reap) | R | Kept verbatim, including reap-after-declare and application scoping. Graph-owned nodes are exempt. | +| `PUT /v1/apps/:app/targets` (sync, reap) | R | Kept, including reap-after-declare and application scoping. The authoritative inventory is read from the durable store as well as the local registry, because a sync may reach a replica before reconcile. An incomplete/refused sync reaps nothing. Multi-node graphs are exempt. | | `GET` target view (4 routes) | R | Fields re-sourced: budgets carry `key`, `count`, `timeMs`, `subWindows`, `value`, `expiresAt`, `utilisation` read live from KV. `utilisation` is still the **worst** counter, now across shared/scoped keys instead of across shards. | | `DELETE` target | R | Store-first, verbatim, including `registered: false` being a success. | | 409 version-bump | R | §12.3, with a much shorter trigger list. | @@ -1638,7 +1657,7 @@ worth keeping. | knob | default | why | |---|---|---| | `GATE_STAGE_BATCH` | 200 | the per-claim batch when a node declares none | -| `GATE_STAGE_CONCURRENCY` | `max(4, partitions)` | worker count per stage | +| `GATE_STAGE_CONCURRENCY` | unset (derive from the node-wide rate) | fleet-wide worker-count override | | `GATE_LEASE_SECONDS` | 30 | the work lease | | `GATE_POLL_TIMEOUT_SECONDS` | 30 | the parked long-poll window | | `GATE_PARK_THRESHOLD_MS` | 1500 | park-vs-release (§6.5) | @@ -1707,7 +1726,8 @@ for a system whose largest declared budget is 400 items a second. workers = clamp(ceil(cap_rate_per_sec / GATE_LANE_CAPACITY), 1, partitions) ``` -from the stage's tightest unscoped budget with its `share` applied. For the three graphs we +from the stage's tightest unconditional, unscoped budget with its `share` applied. A `whenOp` +budget limits only the selected traffic and cannot size the whole node. For the three graphs we run — sixteen stages, caps between 1.7 and 400 items a second — that is **one worker per stage**: sixteen parked polls per replica, about 1,900 pops an hour, against v1's ~275,000. `airbnb` is six of those sixteen, `vrbo` six and `google` four — a stage being one node on one diff --git a/Dockerfile b/Dockerfile index 6c03035..8849f79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ # Run: docker run -p 8788:8788 -e QUEEN_URL=http://queen:6632 gate # ---------------------------------------------------------------- the console -FROM node:22-alpine AS ui-builder +FROM node:24-alpine AS ui-builder WORKDIR /app/ui COPY ui/package*.json ./ diff --git a/README.md b/README.md index 5bfeba5..6594acb 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,10 @@ reaches it — and `GATE_PUBLIC_BIND` requires a Google session on every route. the local sign-in bypass and Gate refuses to boot with it set on an `https` public URL; `GATE_ADMIN_EMAILS` is what makes that identity able to write rather than only read. +Building from source requires Node.js 24 for the embedded console; the root `.nvmrc` selects it. + ```bash +nvm use # Node.js 24, from the root .nvmrc cd ui && npm ci && npm run build && cd .. # the console is compiled into the binary cargo build --release --workspace cargo test --workspace # the live suite reports as ignored @@ -209,6 +212,7 @@ with no broker configured, which is green lines that verified nothing. CI sets | `GATE_MAX_PARK_MS` | 30000 | how long a handler holds its claim waiting for a window before releasing | | `GATE_INTERIOR_SEED_SKEW_SECONDS` | 120 | how far before a graph's start a new group on an **interior** queue is seeded; a margin for Gate's clock against the broker's, capped at 600 | | `GATE_RECONCILE_SECONDS` | 15 | how often a replica re-reads the store | +| `GATE_MAX_PUSH_BODY_BYTES` | 8388608 | the largest body a **push** route buffers, clamped to 2 MiB–64 MiB. 2 MiB is axum's default, which is what applied to everything until 2026-09-04 because nothing set one; the ceiling is there because the limit is a per-request memory reservation and nothing bounds how many requests hold one at once. Document routes keep the default | **Where a new consumer group starts, and it is two rules.** On an **ingress** queue — yours, or Gate's own HTTP front door — a new group is seeded at the *head* of the retained log, because a @@ -231,8 +235,9 @@ there are counters (`popped`, `admitted`, `deferred`, `parked`, `released`, `for explains a stage's throughput. `wedged` is the one to alert on: it counts a stage whose ack the broker keeps refusing at a claim head that never moves, which is a stuck cursor and not a budget backlog — the stage says so once at `ERROR` with the `seek` that fixes it. Denials are kept in a bounded in-process ring; admissions are counted, never -traced. Rollups are opt-in per graph (`"counters": { "windowSeconds": 60 }`), because observability -is a thing you switch on, not a thing that runs whether or not anyone is looking. +traced. Rollups are opt-in per graph (`"counters": { "windowSeconds": 60 }`); the current storage +and API contract is a fixed one-minute window, so `60` is the only accepted value. Observability is +a thing you switch on, not a thing that runs whether or not anyone is looking. **One thing to say out loud.** The declaration names your egress queue, and: diff --git a/crates/core/src/cost.rs b/crates/core/src/cost.rs index 912bd13..f92467c 100644 --- a/crates/core/src/cost.rs +++ b/crates/core/src/cost.rs @@ -7,7 +7,8 @@ use serde_json::Value; -use crate::doc::{Cost, PAYLOAD_ROOT}; +use crate::doc::{Cost, GATE_META, PAYLOAD_ROOT}; +use crate::plan::CompiledBudget; /// Walk a dotted payload path. The first segment must be `payload`, which names /// the message's own `data`; `payload.a.b` is `data["a"]["b"]`. @@ -21,22 +22,30 @@ pub fn resolve<'a>(data: &'a Value, path: &str) -> Option<&'a Value> { if segs.next()? != PAYLOAD_ROOT { return None; } - let mut cur = data; + let first = segs.next()?; + if first.is_empty() || first == GATE_META { + return None; + } + let mut cur = data.get(first)?; for s in segs { + if s.is_empty() { + return None; + } cur = cur.get(s)?; } Some(cur) } /// Whether a string is a usable payload path: `payload` plus at least one -/// segment, each of them non-empty. +/// non-empty segment. Gate's root `_gate` envelope is deliberately outside the +/// declaration language: costs and scopes may only come from producer data. pub fn ok_payload_path(path: &str) -> bool { let mut segs = path.split('.'); if segs.next() != Some(PAYLOAD_ROOT) { return false; } let rest: Vec<&str> = segs.collect(); - !rest.is_empty() && rest.iter().all(|s| !s.is_empty()) + !rest.is_empty() && rest.first() != Some(&GATE_META) && rest.iter().all(|s| !s.is_empty()) } /// The scope value a budget keys on, as it reaches the kv key. @@ -54,6 +63,32 @@ pub fn scope_value(data: &Value, path: &str) -> Option { } } +/// The first applicable scoped budget whose key cannot be resolved. +/// +/// Applicability comes first: a `photo.delete` per-listing budget has no reason +/// to require `listingId` from a `photo.upload`. Both the HTTP door and the +/// relay use this one answer so direct queue ingress cannot enforce a different +/// contract from HTTP ingress. +pub fn missing_scope<'a>( + budgets: &'a [CompiledBudget], + data: &Value, +) -> Option<(&'a str, &'a str)> { + let op = op_of(data); + budgets.iter().find_map(|budget| { + if budget + .when_op + .as_ref() + .is_some_and(|patterns| !op_matches(patterns, op)) + { + return None; + } + let path = budget.scope_by.as_deref()?; + scope_value(data, path) + .is_none() + .then_some((budget.id.as_str(), path)) + }) +} + /// What this item costs, or the reason it can never be admitted. /// /// Integers, because `kv.incr`'s delta is `i64` on this wire. A resolved cost @@ -69,10 +104,22 @@ pub fn cost_of(cost: &Cost, data: &Value) -> Result { Cost::Fixed(n) => (*n, *n), Cost::Path(c) => { let max = c.max.unwrap_or(c.default); - let v = resolve(data, &c.path) - .and_then(integral) - .filter(|n| *n >= 1) - .unwrap_or(c.default); + let v = match resolve(data, &c.path) { + Some(value) => match integral(value) { + Ok(value) => value.filter(|n| *n >= 1).unwrap_or(c.default), + // `i64::MAX` is the largest lower bound the public error + // type can carry. It is enough to refuse every ordinary + // maximum; equality is the sentinel for an out-of-range + // positive number and gets its own truthful message below. + Err(()) => { + return Err(TooExpensive { + cost: i64::MAX, + max, + }) + } + }, + None => c.default, + }; (v, max) } }; @@ -94,6 +141,13 @@ pub struct TooExpensive { impl std::fmt::Display for TooExpensive { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.cost == i64::MAX && self.max == i64::MAX { + return write!( + f, + "this item declares a cost outside the signed 64-bit range the broker can charge: \ + refusing it is safer than silently charging i64::MAX" + ); + } write!( f, "this item declares a cost of {} and the node admits at most {}: an item that cannot \ @@ -105,13 +159,38 @@ impl std::fmt::Display for TooExpensive { } /// A JSON number that is a whole number. `3.0` is three; `3.5` is not a cost. -fn integral(v: &Value) -> Option { +/// +/// A whole number outside `i64` is an error rather than a missing value. Rust's +/// float-to-integer cast saturates, so treating it as an ordinary conversion +/// would collapse every larger JSON number to `i64::MAX` and undercharge it. +fn integral(v: &Value) -> Result, ()> { match v { Value::Number(n) => match n.as_i64() { - Some(i) => Some(i), - None => n.as_f64().filter(|f| f.fract() == 0.0).map(|f| f as i64), + Some(i) => Ok(Some(i)), + None => { + let Some(f) = n.as_f64() else { + return Ok(None); + }; + if f.fract() != 0.0 { + return Ok(None); + } + // `i64::MAX as f64` rounds to 2^63, one past the largest i64, + // so the upper bound is deliberately exclusive. The lower one + // is inclusive because -2^63 is representable. + const I64_BOUND: f64 = 9_223_372_036_854_775_808.0; + if f >= I64_BOUND { + return Err(()); + } + // A negative value already means "use the default". Keep that + // tolerance even when its magnitude is outside i64; unlike an + // oversized positive cost, it cannot make Gate undercharge. + if f < -I64_BOUND { + return Ok(None); + } + Ok(Some(f as i64)) + } }, - _ => None, + _ => Ok(None), } } diff --git a/crates/core/src/doc.rs b/crates/core/src/doc.rs index d17def5..c8faf00 100644 --- a/crates/core/src/doc.rs +++ b/crates/core/src/doc.rs @@ -100,11 +100,16 @@ pub struct Counters { pub window_seconds: u32, } +/// Roll-ups are stored in minute-keyed rows and every history endpoint reads +/// those rows as minutes. Keep the one supported value named in one place so a +/// declaration cannot promise a window the runtime does not actually emit. +pub const COUNTERS_WINDOW_SECONDS: u32 = 60; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] #[serde(deny_unknown_fields)] pub struct Node { - /// At least one, and at least one of them unscoped — see `node-budget` and - /// `node-unscoped-budget`. + /// At least one, and at least one of them unconditional and unscoped — see + /// `node-budget` and `node-unscoped-budget`. #[serde(default)] pub budgets: Vec, @@ -129,9 +134,9 @@ pub struct Node { #[serde(default, skip_serializing_if = "Option::is_none")] pub batch: Option, - /// How many workers drain this node's stages. Defaults to - /// `max(4, source partitions)`. More workers than partitions is harmless - /// (the extras find nothing and park); fewer is a throughput ceiling. + /// How many workers drain this node's stages. By default this is derived + /// from the tightest unconditional, unscoped rate and capped at the source + /// partition count; an explicit value overrides that derivation. #[serde(default, skip_serializing_if = "Option::is_none")] pub concurrency: Option, } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index e855a51..956879a 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -20,11 +20,13 @@ pub mod plan; pub mod v1; pub mod validate; -pub use cost::{cost_of, ok_payload_path, op_matches, op_of, resolve, scope_value, TooExpensive}; +pub use cost::{ + cost_of, missing_scope, ok_payload_path, op_matches, op_of, resolve, scope_value, TooExpensive, +}; pub use doc::{ default_application, ok_name, ok_target_name, Budget, Confidence, Cost, CostPath, Counters, - Egress, EgressSpec, GraphDoc, Ingress, IngressSpec, Node, Path, PathElem, GATE_META, - PAYLOAD_ROOT, + Egress, EgressSpec, GraphDoc, Ingress, IngressSpec, Node, Path, PathElem, + COUNTERS_WINDOW_SECONDS, GATE_META, PAYLOAD_ROOT, }; pub use ids::derive; pub use plan::{ @@ -34,6 +36,7 @@ pub use plan::{ ASSUMED_FACTOR, DEFAULT_BATCH, DEFAULT_INGRESS_PARTITIONS, }; pub use validate::{ - needs_version_bump, validate, validate_with, warnings, warnings_with, ExternalFacts, Problem, - QueueFacts, + needs_version_bump, refuses_stored_document, validate, validate_plan_with, validate_with, + warnings, warnings_with, ExternalFacts, Problem, QueueFacts, MAX_BREAKER_COUNTERS, + MAX_GRAPH_WORKERS, }; diff --git a/crates/core/src/migrate.rs b/crates/core/src/migrate.rs index 9c82f50..07eaa4d 100644 --- a/crates/core/src/migrate.rs +++ b/crates/core/src/migrate.rs @@ -103,7 +103,26 @@ fn rolling_sub_windows(time_ms: i64, count: i64, cost_max: i64) -> u32 { } fn budget(b: &v1::Budget, cost_max: i64, out: &mut Vec, node: &str) -> Budget { - let time_ms = b.period_seconds.max(1) * 1000; + let seconds = b.period_seconds.max(1); + // v1 carries seconds in an i64 while v2 carries milliseconds in one. The + // multiplication can therefore overflow for a document that is perfectly + // valid on the old wire. Keep the migration total and say explicitly when + // the destination type cannot represent the original duration. + let time_ms = seconds.saturating_mul(1000); + if seconds > i64::MAX / 1000 { + out.push(w( + "period-clamped", + format!( + "budget `{}` of node `{node}` declares periodSeconds {}. The v2 `timeMs` field \ + cannot represent that many milliseconds, so it was capped at {}ms instead of \ + overflowing the migration. Lower the period and redeclare if this budget is \ + intended to rotate on an operational timescale.", + b.id, + b.period_seconds, + i64::MAX + ), + )); + } let count = (b.cap.floor() as i64).max(1); let sub_windows = match b.alignment { @@ -125,8 +144,8 @@ fn budget(b: &v1::Budget, cost_max: i64, out: &mut Vec, node: &str) -> admitted.", b.id, (time_ms / n.max(1) as i64) / 1000, - 2 * (count / n.max(1) as i64).max(1), - 2 * count + (count / n.max(1) as i64).max(1).saturating_mul(2), + count.saturating_mul(2) ), )); Some(n) @@ -227,9 +246,9 @@ fn budget(b: &v1::Budget, cost_max: i64, out: &mut Vec, node: &str) -> /// Two v1 shapes land here. A **class node** with an out-edge was allowed to /// declare no budget at all: it existed to isolate a traffic class and carry a /// priority, and the limit it was checked against lived downstream. And a node -/// with only SCOPED budgets was legal too — v1's ETA read the worst key and its -/// breach ring was per-replica, so neither needed a node-level denominator; v2's -/// ETA and its breaker both do. +/// with only SCOPED or CONDITIONAL budgets was legal too — v1's ETA read the +/// worst key and its breach ring was per-replica, so neither needed a counter +/// every item meets; v2's ETA and its breaker both do. /// /// Either way the mapping declares a pass-through — which limits nothing, /// exactly as before — and says so loudly, rather than inventing a ceiling @@ -238,11 +257,12 @@ fn passthrough_budget(node: &str, out: &mut Vec) -> Budget { out.push(w( "node-budget", format!( - "node `{node}` declared no budget on the node itself (v1 allowed that for a class \ - node, and for a node carrying only per-key budgets). v2 requires one — it is what \ - the ETA measures a rate against and what the breaker spends when a vendor says 429 \ - — so a pass-through of 1000000 per second has been declared for it: it limits \ - nothing, exactly as before. Replace it with the real limit." + "node `{node}` declared no unconditional budget on the node itself (v1 allowed that \ + for a class node, and for a node carrying only per-key or conditional budgets). v2 \ + requires one — it is what the ETA measures a rate against and what the breaker \ + spends when a vendor says 429 — so a pass-through of 1000000 per second has been \ + declared for it: it limits nothing, exactly as before. Replace it with the real \ + limit." ), )); Budget { @@ -392,7 +412,11 @@ pub fn from_v1_target(spec: &v1::TargetSpec) -> Result { // NOT mapped — see `lane_concurrency_warning`. concurrency: None, }; - if node.budgets.iter().all(|b| b.scope_by.is_some()) { + if node + .budgets + .iter() + .all(|b| b.scope_by.is_some() || b.when_op.is_some()) + { node.budgets.push(passthrough_budget(&node_name, &mut out)); } lane_concurrency_warning(&lanes, &node_name, &mut out); @@ -480,7 +504,11 @@ pub fn from_v1_graph(spec: &v1::GraphSpec) -> Result { // NOT mapped — see `lane_concurrency_warning`. concurrency: None, }; - if node.budgets.iter().all(|b| b.scope_by.is_some()) { + if node + .budgets + .iter() + .all(|b| b.scope_by.is_some() || b.when_op.is_some()) + { node.budgets.push(passthrough_budget(name, &mut out)); } if node.egress.is_some() { diff --git a/crates/core/src/plan.rs b/crates/core/src/plan.rs index d74747c..68d6c7a 100644 --- a/crates/core/src/plan.rs +++ b/crates/core/src/plan.rs @@ -22,6 +22,7 @@ //! 16 workers per cycle across 2 legs each, a reconcile loop, a history prune //! and a depth cache serving four read shapes. v2 runs seven consumers. +use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use crate::doc::{Confidence, Cost, GraphDoc, Node, Path, PathElem}; @@ -297,6 +298,12 @@ pub struct Destination { /// egress queue, which is why this is recorded in §7 rather than done /// quietly. pub derive_id: bool, + /// Whether the queue this push enters is read by more than one stage. + /// Such a queue needs `_gate.path` on every frame so each consumer group + /// can distinguish its own copy from the other paths' copies. A scalar or + /// array cannot carry that stamp and must be rejected before it reaches + /// this destination rather than being charged and routed as another path. + pub requires_stamp: bool, pub terminal: bool, } @@ -352,7 +359,7 @@ impl CompiledBudget { /// 64 state documents to hold the same thing. pub fn key_for(&self, scope: Option<&str>) -> String { match (self.scope_by.as_ref(), scope) { - (Some(_), Some(v)) => format!("{}:{v}", self.key), + (Some(_), Some(v)) => format!("{}:{}", self.key, budget_key_component(v)), _ => self.key.clone(), } } @@ -395,11 +402,54 @@ pub fn stage_group(app: &str, graph: &str, path: &str, node: &str) -> String { } pub fn budget_key(app: &str, graph: &str, node: &str, bid: &str) -> String { - format!("b:{app}:{graph}:{node}:{bid}") + let graph = local_budget_graph_component(graph); + format!( + "b:{}:{graph}:{}:{}", + budget_key_component(app), + budget_key_component(node), + budget_key_component(bid) + ) } pub fn shared_budget_key(app: &str, shared: &str) -> String { - format!("b:{app}:shared:{shared}") + format!( + "b:{}:shared:{}", + budget_key_component(app), + budget_key_component(shared) + ) +} + +/// Escape the separator and the escape byte in a dynamic KV-key component. +/// +/// Budget ids, shared keys and scope values are deliberately free-form. A raw +/// colon would otherwise make `(id = "a:b")` indistinguishable from +/// `(id = "a", scope = "b")`, causing two declarations to spend one row. +/// Ordinary names keep their historical spelling. +fn budget_key_component(value: &str) -> Cow<'_, str> { + if !value.bytes().any(|b| matches!(b, b'%' | b':')) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for c in value.chars() { + match c { + '%' => escaped.push_str("%25"), + ':' => escaped.push_str("%3A"), + _ => escaped.push(c), + } + } + Cow::Owned(escaped) +} + +/// `shared` is the structural marker immediately after the application in a +/// shared key. It is also a legal graph name, so encode that one local graph +/// spelling to keep the two namespaces disjoint. A literal `%73hared` cannot +/// collide because the general component encoder escapes its percent sign. +fn local_budget_graph_component(graph: &str) -> Cow<'_, str> { + if graph == "shared" { + Cow::Borrowed("%73hared") + } else { + budget_key_component(graph) + } } pub fn breaker_key(app: &str, graph: &str, node: &str) -> String { @@ -640,6 +690,7 @@ pub fn compile_with(doc: &GraphDoc, opts: &PlanOpts) -> Plan { label: egress_label(app, graph, &p.name, node_name), // ALWAYS at a terminal. See the note on `derive_id`. derive_id: true, + requires_stamp: false, terminal: true, }], None => Vec::new(), @@ -654,6 +705,7 @@ pub fn compile_with(doc: &GraphDoc, opts: &PlanOpts) -> Plan { queue: dn.interior_queue.clone(), label: label(app, graph, &p.name, d), derive_id: false, + requires_stamp: false, terminal: false, }) }) @@ -726,6 +778,7 @@ pub fn compile_with(doc: &GraphDoc, opts: &PlanOpts) -> Plan { s.owns_unstamped = claimed.insert(s.source.clone()); let fanout = s.destinations.len() > 1; for d in &mut s.destinations { + d.requires_stamp = readers.get(&d.queue).copied().unwrap_or(1) > 1; d.derive_id = d.terminal || fanout || converging.get(&d.queue).copied().unwrap_or(1) > 1; } @@ -807,10 +860,9 @@ pub fn compile_with(doc: &GraphDoc, opts: &PlanOpts) -> Plan { /// itself by the LEASE, which is the thing this design set out to remove. /// /// So the batch is clamped to what one sub-window admits at the typical item -/// cost: `round(count_sub × share) / cost.default`, over the tightest UNSCOPED -/// budget. Scoped budgets are excluded on purpose — a batch of two hundred -/// messages across two hundred different keys spends one unit of each, and -/// sizing on a per-key count would shrink every claim to a per-key allowance. +/// cost: `round(count_sub × share) / cost.default`, over the tightest NODE-WIDE +/// budget. Scoped and `whenOp` budgets are excluded on purpose: neither is a +/// ceiling every item in the claim necessarily meets. /// /// It is a floor of one and a ceiling of what the declaration asked for: a wide /// budget leaves the declared batch untouched, and a tight one gets a claim that @@ -818,7 +870,7 @@ pub fn compile_with(doc: &GraphDoc, opts: &PlanOpts) -> Plan { fn fitting_batch(np: &NodePlan, share: f64, declared: u32) -> u32 { let per_item = np.cost.default_value().max(1); let fits = np - .unscoped() + .node_wide_rates() .map(|b| (b.max_for(share) / per_item).max(1)) .min(); match fits { @@ -845,9 +897,10 @@ fn fitting_batch(np: &NodePlan, share: f64, declared: u32) -> u32 { /// that can never have work. Stage, measured: ~200 gate consumers for a system /// whose largest declared budget is 400 items a second. /// -/// Only the UNSCOPED budgets, for the same reason `fitting_batch` uses them: a -/// per-key budget is not a rate the node has. *100 photo deletions per listing -/// per week* says nothing about how fast the node drains. +/// Only NODE-WIDE budgets, for the same reason `fitting_batch` uses them: a +/// per-key budget is not a rate the node has, and neither is one selected by +/// `whenOp`. *100 photo deletions per listing per week* says nothing about how +/// fast the whole node drains. /// /// And not the migration's [`PASSTHROUGH_BUDGET_ID`], which is a sentinel and /// not a measurement: a million a second means "this node limits nothing", and @@ -862,8 +915,7 @@ fn fitting_batch(np: &NodePlan, share: f64, declared: u32) -> u32 { fn fitting_workers(np: &NodePlan, share: f64, partitions: u32, lane_capacity: u32) -> u32 { let capacity = lane_capacity.max(1) as f64; let tightest = np - .unscoped() - .filter(|b| b.id != PASSTHROUGH_BUDGET_ID) + .node_wide_rates() .map(|b| b.max_for(share) as f64 / b.window_sub_seconds.max(1) as f64) .fold(f64::INFINITY, f64::min); if !tightest.is_finite() { @@ -1020,13 +1072,38 @@ impl Plan { impl NodePlan { /// The budgets that live on the node itself, rather than one per key. /// - /// The ETA measures a rate against these and the breaker spends them, which - /// is why `node-unscoped-budget` requires at least one: a node with only - /// per-key budgets has no lever and no denominator. + /// The breaker spends these. ETA and other node-wide calculations must + /// additionally exclude budgets with `whenOp`: a conditional counter is + /// not a rate every item meets. pub fn unscoped(&self) -> impl Iterator { self.budgets.iter().filter(|b| !b.is_scoped()) } + /// Budgets every item through the node must meet. These are the only honest + /// denominator for aggregate scheduling, ETA and utilisation: `whenOp` + /// counters apply to a subset whose size those calculations cannot know. + pub fn node_wide(&self) -> impl Iterator { + self.unscoped().filter(|b| b.when_op.is_none()) + } + + /// The node-wide budgets that are a MEASUREMENT, which is a different + /// question from [`NodePlan::node_wide`] and the reason the two are separate. + /// + /// `node-unscoped-budget` asks whether some counter every item meets exists, + /// and the migration's [`PASSTHROUGH_BUDGET_ID`] is there precisely to + /// answer it for a v1 node that declared no limit of its own. But a million + /// a second is a sentinel and not a rate: used as a denominator it answers + /// every question with "there is room" — an ETA of zero, a utilisation near + /// zero, a claim sized against a limit nobody declared. + /// + /// So scheduling, the ETA and utilisation read this one, and a node with + /// nothing here has no node-wide rate at all. That is the honest answer: + /// what such a node forwards is bounded by the tight node downstream of it, + /// not by itself. + pub fn node_wide_rates(&self) -> impl Iterator { + self.node_wide().filter(|b| b.id != PASSTHROUGH_BUDGET_ID) + } + /// The widest ceiling any path can reach at this node — what the breaker /// writes when it spends the window, so no path can slip under it. pub fn widest_share(&self) -> f64 { diff --git a/crates/core/src/validate.rs b/crates/core/src/validate.rs index 5a2ed9a..199afd7 100644 --- a/crates/core/src/validate.rs +++ b/crates/core/src/validate.rs @@ -16,16 +16,35 @@ //! `max-keys`, `store-fits`, `kv-chunk`: cardinality is Postgres rows with a //! TTL, not entries in a document Gate re-reads whole every cycle). -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use crate::cost::ok_payload_path; -use crate::doc::{ok_name, Confidence, Cost, GraphDoc, PathElem}; +use crate::doc::{ok_name, Confidence, Cost, GraphDoc, PathElem, COUNTERS_WINDOW_SECONDS}; use crate::plan::{self, Plan}; /// The largest claim a node may ask for. §12.2's clamp on v1's `pacing.batch`, /// enforced as a refusal. pub const MAX_BATCH: u32 = 1000; +/// The most distinct node-wide counters one breaker can hold. +/// +/// Queen accepts 256 operations in one KV batch and the final operation is the +/// breaker record. Splitting that batch would make the hold and its audit record +/// observably non-atomic. +pub const MAX_BREAKER_COUNTERS: usize = 255; + +/// The largest number of consumer workers one graph may start across all of +/// its stages. +/// +/// `queen-mq` allocates a task and a long-poll loop for every worker before the +/// consumer starts. Without a graph-wide bound, a small declaration containing +/// a large `concurrency` integer can make the process reserve billions of task +/// slots and abort before it can return a refusal. 4096 is already roughly four +/// million items/s at the deliberately pessimistic one-lane capacity; larger +/// deployments should be split into graphs so one declaration cannot exhaust a +/// replica on its own. +pub const MAX_GRAPH_WORKERS: u64 = 4096; + /// The most re-entries a document may allow one item (§16.6). v1's /// `breach-attempts` policed the same number for the same reason. pub const MAX_ATTEMPTS_CEILING: u32 = 20; @@ -56,7 +75,8 @@ impl std::fmt::Display for Problem { #[derive(Debug, Clone, Default)] pub struct ExternalFacts { pub queues: BTreeMap, - /// Ingress queues already claimed elsewhere in the fleet: + /// Stage source queues already claimed elsewhere in the fleet (both + /// ingress and Gate-owned interior queues): /// `(queue, "app/graph", node)`. This graph's own entries must be excluded /// by the caller, or a redeclare would collide with itself. pub ingress_owners: Vec<(String, String, String)>, @@ -79,11 +99,55 @@ fn p(rule: &'static str, detail: String) -> Problem { // -------------------------------------------------------------------- refusals +/// Whether a stored document must be REFUSED for this rule, or may keep running. +/// +/// Every rule here is enforced without exception against a CALLER's declare. +/// The question this answers is a different one: what should happen when a +/// document already in the store violates a rule that did not exist when it was +/// written. +/// +/// Refusing it is not the safe direction. `restore` and `reconcile` both go +/// through the same declare path, so a refusal leaves the graph unregistered: +/// its pushes answer 404, its ingress queue fills with nobody draining it, and +/// the only trace is one WARN line. That is strictly worse than the condition +/// the new rule describes — the graph was serving traffic a moment ago, and the +/// rule was added to stop the NEXT declare, not to stop this one. +/// +/// So a rule stops a stored document only when the plan cannot be built or +/// addressed at all — no nodes, no paths, or a name that cannot become a queue +/// name and a kv key — or when starting it would exhaust the replica before it +/// served anything (`graph-workers`): a WARN written while the process runs out +/// of task slots is not a graph kept running, and it takes every other graph on +/// the replica down with it. Everything else is logged and kept running, and +/// the next caller declare still has to fix it. +pub fn refuses_stored_document(rule: &str) -> bool { + matches!( + rule, + "nodes" + | "paths" + | "application" + | "graph-name" + | "node-name" + | "path-name" + | "graph-workers" + ) +} + pub fn validate(doc: &GraphDoc) -> Vec { validate_with(doc, &ExternalFacts::default()) } pub fn validate_with(doc: &GraphDoc, facts: &ExternalFacts) -> Vec { + let plan = plan::compile(doc); + validate_plan_with(doc, &plan, facts) +} + +/// Validate the exact plan a caller is about to start. +/// +/// Most callers use [`validate_with`]. The server compiles with broker facts +/// and operator overrides first, however, so it must validate that resolved +/// plan rather than silently recompile with library defaults. +pub fn validate_plan_with(doc: &GraphDoc, plan: &Plan, facts: &ExternalFacts) -> Vec { let mut out = Vec::new(); naming(doc, &mut out); if doc.nodes.is_empty() { @@ -100,14 +164,49 @@ pub fn validate_with(doc: &GraphDoc, facts: &ExternalFacts) -> Vec { return out; } - let plan = plan::compile(doc); shape(doc, &mut out); - budgets(doc, &plan, &mut out); - shares(doc, &plan, &mut out); - ownership(doc, facts, &mut out); + counters(doc, &mut out); + worker_width(plan, &mut out); + budgets(doc, plan, &mut out); + shares(doc, plan, &mut out); + ownership(plan, facts, &mut out); out } +fn counters(doc: &GraphDoc, out: &mut Vec) { + let Some(counters) = &doc.counters else { + return; + }; + if counters.window_seconds != COUNTERS_WINDOW_SECONDS { + out.push(p( + "counters-window", + format!( + "counters.windowSeconds is {}, but Gate currently stores and serves fixed \ + one-minute roll-ups. Set windowSeconds to {COUNTERS_WINDOW_SECONDS}, or omit \ + counters to leave durable roll-ups off.", + counters.window_seconds + ), + )); + } +} + +fn worker_width(plan: &Plan, out: &mut Vec) { + let workers: u64 = plan.stages.iter().map(|s| u64::from(s.concurrency)).sum(); + if workers > MAX_GRAPH_WORKERS { + out.push(p( + "graph-workers", + format!( + "this graph resolves to {workers} consumer workers across {} stages; the maximum \ + is {MAX_GRAPH_WORKERS}. Each worker allocates a task and a broker long-poll before \ + the graph starts, so an unbounded value can exhaust a replica. Lower \ + `nodes[].concurrency` or `GATE_STAGE_CONCURRENCY`, or split the topology into \ + separate graphs.", + plan.stages.len() + ), + )); + } +} + fn naming(doc: &GraphDoc, out: &mut Vec) { if !ok_name(&doc.application) { out.push(p( @@ -336,13 +435,27 @@ fn budgets(doc: &GraphDoc, plan: &Plan, out: &mut Vec) { )); continue; } - if np.unscoped().next().is_none() { + if np.node_wide().next().is_none() { out.push(p( "node-unscoped-budget", format!( - "node `{name}` has only per-key budgets. It needs at least one budget on the \ - node itself: it is what the ETA measures a rate against and what the breaker \ - spends when a vendor says 429." + "node `{name}` has no unconditional budget on the node itself. It needs at \ + least one budget without scopeBy or whenOp: every item must meet that \ + counter, so the ETA has a node-wide rate and the breaker can stop every \ + operation when a vendor says 429." + ), + )); + } + let breaker_keys: HashSet<&str> = np.unscoped().map(|b| b.key.as_str()).collect(); + if breaker_keys.len() > MAX_BREAKER_COUNTERS { + out.push(p( + "breaker-width", + format!( + "node `{name}` compiles to {} distinct node-wide counters. A breaker must \ + spend them and write its audit record atomically, but the broker accepts at \ + most 256 operations in one call. Keep at most {MAX_BREAKER_COUNTERS} \ + distinct unscoped counters; budgets with the same sharedKey count once.", + breaker_keys.len() ), )); } @@ -419,7 +532,8 @@ fn budgets(doc: &GraphDoc, plan: &Plan, out: &mut Vec) { "scope-path", format!( "budget `{}` of node `{name}`: scopeBy `{path}` is not a payload path. \ - Write it as `payload.field` or `payload.a.b`.", + Write it as `payload.field` or `payload.a.b`; `payload._gate` is \ + reserved for Gate's routing stamp.", cb.id ), )); @@ -503,7 +617,8 @@ fn cost_rules(name: &str, cost: &Cost, out: &mut Vec) { "cost-path", format!( "node `{name}`: cost.path `{}` is not a payload path. Write it as \ - `payload.field` or `payload.a.b`.", + `payload.field` or `payload.a.b`; `payload._gate` is reserved for Gate's \ + routing stamp.", c.path ), )); @@ -613,41 +728,97 @@ fn shares(doc: &GraphDoc, plan: &Plan, out: &mut Vec) { } } -fn ownership(doc: &GraphDoc, facts: &ExternalFacts, out: &mut Vec) { - // Two nodes in ONE document naming one ingress queue, and the same question - // asked of the fleet. Both are refusals: two consumers of one queue in - // different groups each get every message, which doubles what leaves. +fn ownership(plan: &Plan, facts: &ExternalFacts, out: &mut Vec) { + // One logical node per source queue, including Gate-owned interior queues. + // Looking only at `node.ingress` misses a named ingress that aliases an + // interior queue: two consumers then read the same physical stream under + // different groups even though the document appears to name two queues. let mut mine: HashMap<&str, &str> = HashMap::new(); - for (name, node) in &doc.nodes { - let Some(q) = node - .ingress - .as_ref() - .filter(|i| i.is_enabled()) - .and_then(|i| i.declared_queue()) - else { - continue; - }; - if let Some(other) = mine.insert(q, name.as_str()) { + let mut reported: HashSet<&str> = HashSet::new(); + for stage in &plan.stages { + let q = stage.source.as_str(); + let name = stage.node.as_str(); + if let Some(other) = mine.insert(q, name) { + if other == name || !reported.insert(q) { + continue; + } out.push(p( "ingress-owner", format!( - "`{q}` is the ingress of both `{other}` and `{name}` in this graph. Two \ - consumers of one queue in different groups each get every message, which \ - doubles what leaves." + "`{q}` is the source of both `{other}` and `{name}` in this graph. One queue \ + cannot be both a declared ingress and a Gate-owned interior stream: their \ + consumer groups would each receive and forward the same messages." ), )); } + } + + // The same source ownership rule across replicas. The facts include every + // source from the local registry; the caller separately checks the durable + // store for declarations this replica has not reconciled yet. + for (q, name) in &mine { if let Some((_, g, n)) = facts.ingress_owners.iter().find(|(oq, _, _)| oq == q) { out.push(p( "ingress-owner", format!( - "`{q}` is already the ingress of node `{n}` in graph `{g}`. Two consumers of \ - one queue in different groups each get every message, which doubles what \ - leaves." + "`{q}` is already the source of node `{n}` in graph `{g}`. Node `{name}` \ + would consume the same physical stream under a different group, so both \ + graphs would forward every message." ), )); } } + + // A terminal destination that is also one of this graph's sources is a + // physical cycle even when the node DAG is acyclic. The simplest case is a + // one-node target whose ingress and egress names are equal: each admitted + // message is atomically pushed back into the queue it was just acked from + // and circulates for ever, paying the budget on every turn. + // Reachability, not membership. A queue that is both an egress and a source + // is only a cycle if work put there can come BACK to it: `in -> mid -> out` + // spread over two paths makes `mid` a terminal destination of one and the + // source of the other, and that is a chain, not a loop. Walk the queue graph + // forward from each terminal destination and look for the queue itself. + let mut forward: HashMap<&str, Vec<&str>> = HashMap::new(); + for stage in &plan.stages { + forward + .entry(stage.source.as_str()) + .or_default() + .extend(stage.destinations.iter().map(|d| d.queue.as_str())); + } + let returns_to = |start: &str| -> bool { + let mut seen: HashSet<&str> = HashSet::new(); + let mut queue: VecDeque<&str> = forward.get(start).into_iter().flatten().copied().collect(); + while let Some(next) = queue.pop_front() { + if next == start { + return true; + } + if !seen.insert(next) { + continue; + } + queue.extend(forward.get(next).into_iter().flatten().copied()); + } + false + }; + let mut feedback: HashSet<&str> = HashSet::new(); + for stage in &plan.stages { + for destination in &stage.destinations { + if destination.terminal + && returns_to(destination.queue.as_str()) + && feedback.insert(destination.queue.as_str()) + { + out.push(p( + "queue-cycle", + format!( + "`{}` is both an egress and a source in this graph. An admitted message \ + would be pushed back into a queue this graph consumes and circulate for \ + ever, paying its budgets on every turn. Use a distinct egress queue.", + destination.queue + ), + )); + } + } + } } // -------------------------------------------------------------------- warnings @@ -673,8 +844,12 @@ pub fn warnings_with(doc: &GraphDoc, facts: &ExternalFacts) -> Vec { // RATE than declared**, never a looser one, which is the safe way to // be wrong — but neither is what the caller wrote down, so both are // said out loud. - let enforced_ms = cb.window_sub_seconds * 1000 * cb.sub_windows as i64; - if enforced_ms != b.time_ms { + // A valid declaration may use the whole i64 range for `timeMs`. + // Rounding each sub-window up can make the reconstructed duration + // slightly larger than i64::MAX, so compare in i128 instead of + // panicking while preparing a warning after the graph is live. + let enforced_ms = (cb.window_sub_seconds as i128) * 1000 * (cb.sub_windows as i128); + if enforced_ms != b.time_ms as i128 { out.push(p( "window-sub-second", format!( diff --git a/crates/core/tests/migrate.rs b/crates/core/tests/migrate.rs index 0be7e8b..d022432 100644 --- a/crates/core/tests/migrate.rs +++ b/crates/core/tests/migrate.rs @@ -83,6 +83,28 @@ fn a_v1_target_becomes_a_one_node_graph_that_validates() { ); } +/// A v1 period is seconds in an i64; v2 stores milliseconds in the same width. +/// Multiplying an otherwise valid old document used to panic in debug builds +/// (and wrap in release builds) before the migration could answer. +#[test] +fn an_extreme_v1_period_is_clamped_and_named_instead_of_overflowing() { + let mut spec: v1::TargetSpec = serde_json::from_str(V1_TARGET).unwrap(); + spec.budgets[0].period_seconds = i64::MAX; + spec.budgets[0].cap = i64::MAX as f64; + + let migrated = migrate::from_v1_target(&spec).expect("the old shape remains readable"); + + assert_eq!(migrated.doc.nodes["airbnb"].budgets[0].time_ms, i64::MAX); + assert!( + rules(&migrated.warnings).contains(&"period-clamped"), + "the unavoidable loss of range must be visible: {:#?}", + migrated.warnings + ); + // The resolved-warning pass is part of the PUT response and used to have + // a second overflow of its own (fixed by the base PR). + let _ = gate_core::warnings(&migrated.doc); +} + /// §12.4's stability promise, which is the whole reason a migration is a /// migration and not a rewrite: the queue the application's consumers are /// already popping does not move. @@ -129,6 +151,24 @@ fn a_class_node_with_no_budget_gets_a_passthrough_and_a_warning() { assert!(rules(&m.warnings).contains(&"node-budget")); } +/// A v1 selector deliberately lets non-matching operations pass. Preserve that +/// behavior, but add the unconditional lever v2's node-wide breaker requires. +#[test] +fn a_node_with_only_conditional_budgets_gets_a_passthrough() { + let mut spec: v1::TargetSpec = serde_json::from_str(V1_TARGET).unwrap(); + spec.budgets[0].matcher = Some(v1::Match { + op: vec!["photo.delete".into()], + }); + + let m = migrate::from_v1_target(&spec).unwrap(); + let budgets = &m.doc.nodes["airbnb"].budgets; + assert_eq!(budgets.len(), 2); + assert_eq!(budgets[1].id.as_deref(), Some("passthrough")); + assert!(budgets[1].when_op.is_none()); + assert!(gate_core::validate(&m.doc).is_empty()); + assert!(rules(&m.warnings).contains(&"node-budget")); +} + #[test] fn every_dropped_v1_field_is_named_in_a_warning() { let spec: v1::GraphSpec = serde_json::from_str(V1_GRAPH).unwrap(); diff --git a/crates/core/tests/plan.rs b/crates/core/tests/plan.rs index d20a6fb..32517a4 100644 --- a/crates/core/tests/plan.rs +++ b/crates/core/tests/plan.rs @@ -236,6 +236,37 @@ fn a_convergence_derives_even_without_a_fanout() { ); } +/// Ownership on a shared interior queue is encoded in `_gate.path`. The +/// immediate writers must therefore know that an unstampable payload cannot be +/// forwarded there; unrelated fan-out branches and linear hops do not inherit +/// that restriction. +#[test] +fn only_destinations_with_multiple_readers_require_a_path_stamp() { + let p = compile(&airbnb()); + let photos = p.stage("photos", "photos").unwrap(); + let ip = photos.destinations.iter().find(|d| d.node == "ip").unwrap(); + let audit = photos + .destinations + .iter() + .find(|d| d.node == "audit") + .unwrap(); + assert!(ip.requires_stamp, "three path groups read ip.in"); + assert!(!audit.requires_stamp, "only one path group reads audit.in"); + + let chain: gate_core::GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{"one":{"ingress":true,"budgets":[{"id":"b","count":100,"timeMs":1000}]}, + "two":{"budgets":[{"id":"b","count":100,"timeMs":1000}],"egress":"a.g.out"}}, + "paths":[{"name":"main","nodes":["one","two"]}]}"#, + ) + .unwrap(); + let chain = compile(&chain); + assert!( + !chain.stage("main", "one").unwrap().destinations[0].requires_stamp, + "one reader needs no ownership stamp" + ); +} + // ---------------------------------------------------------------- ceilings /// Priority is a per-path `max` on ONE counter. The top half of `ip` is an @@ -489,6 +520,76 @@ fn a_scoped_budget_keys_on_the_value() { assert_eq!(b.key_for(None), "b:channel:airbnb:photos:per-listing"); } +#[test] +fn a_scope_is_required_only_when_its_budget_applies() { + let p = compile(&airbnb()); + let budgets = &p.node("photos").unwrap().budgets; + + assert_eq!( + gate_core::missing_scope( + budgets, + &serde_json::json!({ "op": "photo.delete", "rooms": 1 }) + ), + Some(("per-listing", "payload.listingId")) + ); + assert_eq!( + gate_core::missing_scope( + budgets, + &serde_json::json!({ "op": "photo.upload", "rooms": 1 }) + ), + None, + "a non-matching whenOp must not require this budget's scope" + ); + assert_eq!( + gate_core::missing_scope( + budgets, + &serde_json::json!({ + "op": "photo.delete", "rooms": 1, "listingId": "l-42" + }) + ), + None + ); +} + +#[test] +fn budget_key_components_cannot_smuggle_separators() { + let mut doc = airbnb(); + doc.nodes.get_mut("photos").unwrap().budgets[1].id = Some("per:listing%v2".into()); + let p = compile(&doc); + let b = p + .node("photos") + .unwrap() + .budgets + .iter() + .find(|b| b.id == "per:listing%v2") + .unwrap(); + + assert_eq!( + b.key_for(Some("listing:42%blue")), + "b:channel:airbnb:photos:per%3Alisting%25v2:listing%3A42%25blue" + ); + assert_ne!( + plan::budget_key("channel", "airbnb", "photos", "per:listing"), + format!( + "{}:listing", + plan::budget_key("channel", "airbnb", "photos", "per") + ) + ); +} + +#[test] +fn local_and_shared_budget_namespaces_cannot_collide() { + let local = plan::budget_key("channel", "shared", "vendor", "minute"); + let shared_scoped = format!("{}:minute", plan::shared_budget_key("channel", "vendor")); + assert_eq!(local, "b:channel:%73hared:vendor:minute"); + assert_ne!(local, shared_scoped); + + assert_ne!( + plan::shared_budget_key("channel", "vendor:minute"), + shared_scoped + ); +} + // ---------------------------------------------------------------------- cost #[test] @@ -520,6 +621,38 @@ fn cost_is_a_payload_path() { assert!(gate_core::cost_of(&cost, &serde_json::json!({"rooms": 51})).is_err()); } +#[test] +fn a_cost_outside_the_brokers_integer_range_is_refused_instead_of_saturated() { + let cost = gate_core::Cost::Path(gate_core::CostPath { + path: "payload.rooms".into(), + default: 1, + max: Some(i64::MAX), + }); + + assert_eq!( + gate_core::cost_of(&cost, &serde_json::json!({ "rooms": i64::MAX })).unwrap(), + i64::MAX, + "the actual wire boundary remains valid" + ); + + for value in [serde_json::json!(u64::MAX), serde_json::json!(1.0e100)] { + let error = gate_core::cost_of(&cost, &serde_json::json!({ "rooms": value })) + .expect_err("an unrepresentable cost must not be charged as i64::MAX"); + assert!( + error + .to_string() + .contains("outside the signed 64-bit range"), + "{error}" + ); + } + + assert_eq!( + gate_core::cost_of(&cost, &serde_json::json!({ "rooms": -1.0e100 })).unwrap(), + 1, + "negative costs retain the documented fallback-to-default behaviour" + ); +} + #[test] fn a_payload_path_must_start_at_the_payload_root() { assert!(gate_core::ok_payload_path("payload.a")); @@ -527,10 +660,17 @@ fn a_payload_path_must_start_at_the_payload_root() { assert!(!gate_core::ok_payload_path("payload")); assert!(!gate_core::ok_payload_path("data.a")); assert!(!gate_core::ok_payload_path("payload.")); - // `_gate` is Gate's own stamp and must stay unaddressable from a document. - assert!( - gate_core::resolve(&serde_json::json!({"_gate": {"path": "x"}}), "_gate.path").is_none() - ); + // `_gate` is Gate's own root stamp and must stay unaddressable from a + // document even through the otherwise-required `payload` prefix. + assert!(!gate_core::ok_payload_path("payload._gate.path")); + assert!(gate_core::resolve( + &serde_json::json!({"_gate": {"path": "x"}}), + "payload._gate.path" + ) + .is_none()); + // A producer may still use the same spelling below another object. Only + // the root key is reserved for Gate. + assert!(gate_core::ok_payload_path("payload.vendor._gate")); } #[test] @@ -798,3 +938,40 @@ const VRBO: &str = r#" const PATHS_WITH_REVIEWS: &str = r#""paths": [ { "name": "reviews", "nodes": ["reviews", "partner"] },"#; + +/// The migration's passthrough is a sentinel, not a measurement, and every +/// node-wide aggregate has to agree about that. `fitting_workers` always did; +/// the batch, the ETA and the flow ceiling reached it through `node_wide` and +/// would have read a million a second as this node's real rate. +#[test] +fn the_passthrough_sentinel_is_not_a_node_wide_rate() { + let mut doc: gate_core::GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{"n":{"ingress":true,"egress":"a.g.out","budgets":[ + {"id":"real","count":100,"timeMs":1000,"whenOp":["listing.update"]}]}}, + "paths":[{"name":"main","nodes":["n"]}]}"#, + ) + .unwrap(); + // What the v1 migration adds to a node carrying only conditional budgets. + doc.nodes.get_mut("n").unwrap().budgets.push( + serde_json::from_str(r#"{"id":"passthrough","count":1000000,"timeMs":1000}"#).unwrap(), + ); + + let p = compile(&doc); + let np = p.node("n").unwrap(); + assert_eq!( + np.node_wide().count(), + 1, + "`node-unscoped-budget` is satisfied: the sentinel IS met by every item" + ); + assert_eq!( + np.node_wide_rates().count(), + 0, + "but it is not a rate, so no aggregate may divide by it" + ); + assert!( + gate_core::validate(&doc).is_empty(), + "a migrated node must still declare clean: {:?}", + gate_core::validate(&doc) + ); +} diff --git a/crates/core/tests/validate.rs b/crates/core/tests/validate.rs index dc8dfd2..0157d01 100644 --- a/crates/core/tests/validate.rs +++ b/crates/core/tests/validate.rs @@ -36,7 +36,10 @@ mod common; use common::{airbnb, rules}; use gate_core::doc::{Egress, Ingress, PathElem}; -use gate_core::{validate, warnings, GraphDoc}; +use gate_core::{ + compile_with, validate, validate_plan_with, warnings, Counters, ExternalFacts, GraphDoc, + PlanOpts, MAX_GRAPH_WORKERS, +}; /// The single most valuable test in the file: the flagship fixture must validate /// clean, in the new vocabulary. If it cannot, the schema is wrong. @@ -82,6 +85,17 @@ fn broken(f: impl FnOnce(&mut GraphDoc)) -> Vec<&'static str> { rules(&validate(&doc)) } +#[test] +fn counters_only_accept_the_window_the_runtime_emits() { + assert!( + broken(|d| d.counters = Some(Counters { window_seconds: 30 })).contains(&"counters-window") + ); + assert!( + !broken(|d| d.counters = Some(Counters { window_seconds: 60 })) + .contains(&"counters-window") + ); +} + // -------------------------------------------------------------------- naming #[test] @@ -189,6 +203,42 @@ fn a_node_needs_at_least_one_unscoped_budget() { assert!(got.contains(&"node-unscoped-budget"), "{got:?}"); } +/// A conditional counter is not a breaker lever for operations it does not +/// select. At least one unscoped counter must therefore take every item. +#[test] +fn a_node_needs_an_unconditional_unscoped_budget() { + let got = broken(|d| { + let n = d.nodes.get_mut("audit").unwrap(); + n.budgets[0].when_op = Some(vec!["photo.delete".into()]); + }); + assert!(got.contains(&"node-unscoped-budget"), "{got:?}"); +} + +#[test] +fn a_breaker_must_fit_its_counters_and_record_in_one_atomic_call() { + let fill = |d: &mut GraphDoc, count: usize| { + let n = d.nodes.get_mut("audit").unwrap(); + let template = n.budgets[0].clone(); + n.budgets = (0..count) + .map(|i| { + let mut b = template.clone(); + b.id = Some(format!("breaker-{i}")); + b + }) + .collect(); + }; + + let mut at_limit = airbnb(); + fill(&mut at_limit, gate_core::MAX_BREAKER_COUNTERS); + assert!( + !rules(&validate(&at_limit)).contains(&"breaker-width"), + "255 counters plus the record must fit exactly" + ); + + let got = broken(|d| fill(d, gate_core::MAX_BREAKER_COUNTERS + 1)); + assert!(got.contains(&"breaker-width"), "{got:?}"); +} + #[test] fn a_budget_that_cannot_admit_anything_never_will() { assert!( @@ -205,6 +255,14 @@ fn the_window_floor_is_a_hundred_milliseconds() { ); } +#[test] +fn warning_about_an_extreme_window_does_not_overflow() { + let mut doc = airbnb(); + doc.nodes.get_mut("audit").unwrap().budgets[0].time_ms = i64::MAX; + let got = rules(&warnings(&doc)); + assert!(got.contains(&"window-sub-second"), "{got:?}"); +} + #[test] fn one_id_declared_twice_would_spend_one_counter() { let got = broken(|d| { @@ -277,6 +335,18 @@ fn a_cost_path_is_a_payload_path() { assert!(got.contains(&"cost-path"), "{got:?}"); } +#[test] +fn a_cost_path_cannot_read_gates_provenance_stamp() { + let got = broken(|d| { + d.nodes.get_mut("audit").unwrap().cost = gate_core::Cost::Path(gate_core::CostPath { + path: "payload._gate.hop".into(), + default: 1, + max: Some(10), + }) + }); + assert!(got.contains(&"cost-path"), "{got:?}"); +} + #[test] fn a_scope_path_is_a_payload_path() { let got = broken(|d| { @@ -285,6 +355,14 @@ fn a_scope_path_is_a_payload_path() { assert!(got.contains(&"scope-path"), "{got:?}"); } +#[test] +fn a_scope_path_cannot_key_on_gates_provenance_stamp() { + let got = broken(|d| { + d.nodes.get_mut("photos").unwrap().budgets[1].scope_by = Some("payload._gate.path".into()) + }); + assert!(got.contains(&"scope-path"), "{got:?}"); +} + /// One counter, two declarations that disagree: one of them is a lie about what /// it enforces. #[test] @@ -380,6 +458,40 @@ fn an_ingress_queue_claimed_elsewhere_in_the_fleet_is_refused() { assert!(got.contains(&"ingress-owner"), "{got:?}"); } +/// A named ingress can spell one of Gate's derived interior names. It is still +/// one physical queue, so treating the two appearances as different logical +/// sources duplicates the stream under two consumer groups. +#[test] +fn a_named_ingress_may_not_alias_an_interior_queue() { + let got = broken(|d| { + d.nodes.get_mut("messages").unwrap().ingress = + Some(Ingress::Named(gate_core::IngressSpec { + queue: Some("gate.channel.airbnb.ip.in".into()), + partitions: None, + http: None, + shed: None, + })); + }); + assert!(got.contains(&"ingress-owner"), "{got:?}"); +} + +/// Node-cycle validation cannot see a loop made only by physical queue names. +/// Without this refusal the relay acks each input and atomically pushes its +/// output back into the same queue, for ever. +#[test] +fn an_egress_may_not_feed_a_source_of_the_same_graph() { + let doc: GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{"n":{"ingress":{"queue":"loop"}, + "budgets":[{"id":"b","count":100,"timeMs":1000}], + "egress":"loop"}}, + "paths":[{"name":"main","nodes":["n"]}]}"#, + ) + .unwrap(); + let got = rules(&validate(&doc)); + assert!(got.contains(&"queue-cycle"), "{got:?}"); +} + // ------------------------------------------------------------------ warnings /// A kv TTL is whole seconds, so a window declared under one is enforced at one @@ -524,6 +636,49 @@ fn a_batch_has_a_range_and_a_scoped_budget_is_why() { ); } +/// `queen-mq` preallocates a vector and spawns one task for every resolved +/// worker. Before this rule, `concurrency: 4294967295` could abort the process +/// while handling a tiny, otherwise valid declaration. +#[test] +fn a_graph_has_a_bounded_total_worker_width() { + let mut doc: GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{"n":{"ingress":true,"concurrency":4096, + "budgets":[{"count":100,"timeMs":1000}], + "egress":"a.g.out"}}, + "paths":[{"name":"main","nodes":["n"]}]}"#, + ) + .unwrap(); + assert_eq!(MAX_GRAPH_WORKERS, 4096); + assert!(!rules(&validate(&doc)).contains(&"graph-workers")); + + doc.nodes.get_mut("n").unwrap().concurrency = Some(4097); + assert!(rules(&validate(&doc)).contains(&"graph-workers")); +} + +/// The server compiles with `GATE_STAGE_CONCURRENCY`; validation must inspect +/// that exact plan rather than recompiling the document with default options. +#[test] +fn a_resolved_global_worker_override_is_bounded_too() { + let doc: GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{"n":{"ingress":true, + "budgets":[{"count":100,"timeMs":1000}], + "egress":"a.g.out"}}, + "paths":[{"name":"main","nodes":["n"]}]}"#, + ) + .unwrap(); + let plan = compile_with( + &doc, + &PlanOpts { + concurrency: Some(4097), + ..Default::default() + }, + ); + let got = validate_plan_with(&doc, &plan, &ExternalFacts::default()); + assert!(rules(&got).contains(&"graph-workers"), "{got:#?}"); +} + #[test] fn a_shared_egress_queue_is_legal_and_named() { let facts = gate_core::ExternalFacts { @@ -562,3 +717,84 @@ fn ingress_true_is_a_queue_gate_owns() { assert!(doc.nodes["prices"].ingress.as_ref().unwrap().http()); assert!(!doc.nodes["messages"].ingress.as_ref().unwrap().http()); } + +/// A rule added after a document was written must not be the thing that takes +/// that document down on the next restart. Only a document whose plan cannot be +/// built or addressed at all is refused on the way back out of the store. +#[test] +fn a_stored_document_is_refused_only_for_a_rule_it_cannot_be_served_under() { + for fatal in [ + "nodes", + "paths", + "application", + "graph-name", + "node-name", + "path-name", + // Not a naming or emptiness rule: a plan over the worker cap would + // exhaust the replica before it served anything, which is not a graph + // kept running and takes every other graph on the replica down too. + "graph-workers", + ] { + assert!( + gate_core::refuses_stored_document(fatal), + "`{fatal}` leaves no plan that can run and must still refuse" + ); + } + for kept in [ + "node-unscoped-budget", + "budget-count", + "cost-fits", + "shares", + "ingress-owner", + "counters-window", + "breaker-width", + "queue-cycle", + "a-rule-that-does-not-exist-yet", + ] { + assert!( + !gate_core::refuses_stored_document(kept), + "`{kept}` would strand a graph that was serving traffic a moment ago" + ); + } +} + +/// A queue that is both an egress and a source is only a cycle when work put +/// there can come back to it. Two paths chained through one queue — `in` to +/// `mid`, then `mid` to `out` — is a legal linear topology, and rejecting it +/// would also stop the graph from restarting on the next boot. +#[test] +fn a_chain_through_one_queue_is_not_a_cycle() { + let chain: GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{ + "first": {"ingress":{"queue":"app.in"}, + "budgets":[{"id":"b1","count":100,"timeMs":1000}], + "egress":"app.mid"}, + "second":{"ingress":{"queue":"app.mid"}, + "budgets":[{"id":"b2","count":100,"timeMs":1000}], + "egress":"app.out"}}, + "paths":[{"name":"p1","nodes":["first"]}, + {"name":"p2","nodes":["second"]}]}"#, + ) + .unwrap(); + assert!( + !rules(&validate(&chain)).contains(&"queue-cycle"), + "`app.mid` is a hop, not a loop: {:?}", + validate(&chain) + ); + + // The real thing: what a node admits goes straight back to what it reads. + let loop_doc: GraphDoc = serde_json::from_str( + r#"{"application":"a","graph":"g","version":1, + "nodes":{"n":{"ingress":{"queue":"app.in"}, + "budgets":[{"id":"b","count":100,"timeMs":1000}], + "egress":"app.in"}}, + "paths":[{"name":"main","nodes":["n"]}]}"#, + ) + .unwrap(); + assert!( + rules(&validate(&loop_doc)).contains(&"queue-cycle"), + "{:?}", + validate(&loop_doc) + ); +} diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index adba32b..6b52eb7 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -22,6 +22,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } parking_lot = "0.12" jsonwebtoken = "9" +getrandom = "0.4" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } rust-embed = "8" mime_guess = "2" diff --git a/crates/server/src/api/breaker.rs b/crates/server/src/api/breaker.rs index 98814d2..7581211 100644 --- a/crates/server/src/api/breaker.rs +++ b/crates/server/src/api/breaker.rs @@ -77,7 +77,17 @@ async fn do_reset(st: &Shared, rt: &std::sync::Arc, node: &str) -> format!("no node `{node}` in graph `{}`", rt.key()), ) })?; - ok(crate::breaker::reset(&st.budgets, np) + let out = crate::breaker::reset(&st.budgets, np) .await - .map_err(|e| Fail(StatusCode::BAD_GATEWAY, e.to_string()))?) + .map_err(|e| Fail(StatusCode::BAD_GATEWAY, e.to_string()))?; + if out.get("ok").and_then(|v| v.as_bool()) == Some(false) { + return Err(Fail( + StatusCode::UNPROCESSABLE_ENTITY, + out.get("error") + .and_then(|v| v.as_str()) + .unwrap_or("the breaker could not be reset") + .to_string(), + )); + } + ok(out) } diff --git a/crates/server/src/api/console.rs b/crates/server/src/api/console.rs index 1451564..85cd87c 100644 --- a/crates/server/src/api/console.rs +++ b/crates/server/src/api/console.rs @@ -13,10 +13,29 @@ use std::collections::{BTreeSet, HashMap}; use axum::extract::Path as AxPath; use axum::extract::{Query, State}; +use axum::http::StatusCode; use serde::Deserialize; use serde_json::{json, Value}; -use crate::api::{ok, ApiResult, Shared}; +use crate::api::{ok, ApiResult, Fail, Shared}; + +/// Console reads are interactive views, not an unbounded export surface. +/// Keeping the bounds here also makes every `usize -> i64/u32` conversion +/// below safe on both 32- and 64-bit builds. +const MAX_HISTORY_MINUTES: usize = 24 * 60; +const MAX_TRACE_ROWS: usize = crate::obs::TRACE_RING; +const MAX_BREACH_ROWS: usize = 500; + +fn bounded(value: Option, default: usize, max: usize) -> usize { + value.unwrap_or(default).clamp(1, max) +} + +fn history_failure(error: String) -> Fail { + Fail( + StatusCode::BAD_GATEWAY, + format!("could not read history: {error}"), + ) +} // ------------------------------------------------------------------ overview @@ -50,6 +69,24 @@ pub async fn overview(State(app): State) -> ApiResult { } } + // Every other field in this document comes from the registry and the + // broker, not from Postgres. Failing the whole response on the history read + // would take `queen.reachable` down with it — the one field that says + // whether the LIMITER is healthy, hidden at exactly the moment an operator + // is working out what broke. So the rate reports itself as unknown and + // names its own failure, and the rest of the overview still answers. + let (admitted_per_sec, history_error) = if counters_on { + match rate_of(&app, now).await { + Ok(rate) => (rate, Value::Null), + Err(Fail(_, message)) => { + tracing::warn!(error = %message, "overview: the admission rate is unreadable"); + (Value::Null, Value::String(message)) + } + } + } else { + (Value::Null, Value::Null) + }; + ok(json!({ "queen": { "reachable": health.reachable, @@ -63,7 +100,10 @@ pub async fn overview(State(app): State) -> ApiResult { // Null, not a lifetime average. A running counter divided by an uptime // the caller does not know is a number that is wrong in a way nobody can // see. - "admitted_per_sec": if counters_on { rate_of(&app, now).await } else { Value::Null }, + "admitted_per_sec": admitted_per_sec, + // Present only when the rate above is null BECAUSE the read failed, so + // a console can tell that apart from roll-ups being switched off. + "history_error": history_error, "budgets_assumed": assumed, "budgets_stale": stale, })) @@ -85,9 +125,9 @@ fn is_stale(as_of: Option<&str>, now_ms: i64) -> bool { } } -async fn rate_of(app: &Shared, now: i64) -> Value { +async fn rate_of(app: &Shared, now: i64) -> Result { let Some(h) = app.history.as_ref() else { - return Value::Null; + return Ok(Value::Null); }; let mut total = 0.0f64; for g in app.registry.all() { @@ -99,10 +139,11 @@ async fn rate_of(app: &Shared, now: i64) -> Value { &s.stage.path, now, ) - .await; + .await + .map_err(history_failure)?; } } - json!(total) + Ok(json!(total)) } // -------------------------------------------------------------------- lists @@ -139,18 +180,26 @@ pub async fn list_targets(State(app): State) -> ApiResult { let now = crate::now_ms(); let mut out = Vec::new(); - for g in app.registry.all() { - // The backlog of everything that has not been admitted yet: every - // ingress queue this graph reads, under the group that reads it. + let graphs = app.registry.all(); + // This is the one route that samples, so it is the one place that knows + // which graphs still exist. + app.backlogs + .retain(&graphs.iter().map(|g| g.key()).collect()); + + for g in graphs { + // The backlog of everything that has not been admitted yet: every stage + // source this graph reads, under the group that reads it. Interior + // stages matter too — a downstream budget can be the binding one. let mut backlog = 0u64; - for s in g.stages.iter().filter(|s| s.stage.first_hop) { + for s in &g.stages { backlog += app .depths .pending_of_group(&app.queen, &s.stage.source, &s.stage.group) - .await + .await? .values() .sum::(); } + let saturating = app.backlogs.sample(&g.key(), backlog); let (mut adm, mut den) = (0u64, 0u64); for s in &g.stages { @@ -169,7 +218,7 @@ pub async fn list_targets(State(app): State) -> ApiResult { let mut worst = (String::new(), 0.0f64, 0i64, 0i64, false, 0i64); for np in g.plan.nodes.values() { let keys: Vec = np.unscoped().map(|b| b.key.clone()).collect(); - let states = app.budgets.read(&keys).await.unwrap_or_default(); + let states = app.budgets.read(&keys).await?; for b in np.unscoped() { let ceiling = b.max_for(np.widest_share()); if ceiling <= 0 { @@ -228,7 +277,15 @@ pub async fn list_targets(State(app): State) -> ApiResult { "worst_assumed": worst.4, "admitted": adm, "denied": den, - "state": if den > 0 { "pacing" } else { "flowing" }, + "state": if !g.is_running() { + "down" + } else if saturating { + "saturating" + } else if backlog > 0 { + "pacing" + } else { + "flowing" + }, "backlog": backlog, "at": now, })); @@ -248,7 +305,7 @@ pub async fn list_graphs(State(st): State) -> ApiResult { waiting += st .depths .pending_of_group(&st.queen, &s.stage.source, &s.stage.group) - .await + .await? .values() .sum::(); admitted += s @@ -310,20 +367,20 @@ pub struct FlowQuery { /// much did we send" but "how close are we to being refused". pub async fn flow(State(app): State, Query(q): Query) -> ApiResult { let now = crate::now_ms(); - let minutes = q.minutes.unwrap_or(120).clamp(1, 1440) as i64; + let minutes = bounded(q.minutes, 120, MAX_HISTORY_MINUTES) as i64; let Some(h) = app.history.as_ref() else { return ok(json!({ "minutes": [], "applications": [], "durable": false })); }; - // The ceiling per node, from the DECLARATION rather than from the data: it - // is what the counter enforces, and a node that admitted nothing this minute - // still has one. + // The node-wide ceiling from the DECLARATION rather than from the data. A + // `whenOp` counter covers only an unknown subset of these admissions, so it + // cannot be the denominator for the whole node. let mut ceiling: HashMap<(String, String), f64> = HashMap::new(); for g in app.registry.all() { for (name, np) in &g.plan.nodes { let per_min = np - .unscoped() + .node_wide_rates() .map(|b| b.count_sub as f64 * 60.0 / b.window_sub_seconds.max(1) as f64) .fold(f64::INFINITY, f64::min); if per_min.is_finite() && per_min > 0.0 { @@ -342,20 +399,23 @@ pub async fn flow(State(app): State, Query(q): Query) -> ApiR utilisation: f64, target: String, admitted: i64, + cost: f64, ceiling: f64, total: i64, } let mut cells: HashMap<(String, i64), Cell> = HashMap::new(); let mut minute_set: BTreeSet = BTreeSet::new(); - for (application, target, minute, admitted) in h.flow(minutes, now).await { + let history = h.flow(minutes, now).await.map_err(history_failure)?; + for (application, target, minute, admitted, cost) in history { minute_set.insert(minute); let cap = ceiling.get(&(application.clone(), target.clone())).copied(); - let u = cap.map_or(0.0, |c| admitted as f64 / c); + let u = cap.map_or(0.0, |c| cost / c); let e = cells.entry((application.clone(), minute)).or_insert(Cell { utilisation: 0.0, target: target.clone(), admitted: 0, + cost: 0.0, ceiling: cap.unwrap_or(0.0), total: 0, }); @@ -364,6 +424,7 @@ pub async fn flow(State(app): State, Query(q): Query) -> ApiR e.utilisation = u; e.target = target; e.admitted = admitted; + e.cost = cost; e.ceiling = cap.unwrap_or(0.0); } } @@ -378,12 +439,14 @@ pub async fn flow(State(app): State, Query(q): Query) -> ApiR .map(|t| match cells.get(&(a.clone(), *t)) { Some(c) => json!({ "t": t, "utilisation": c.utilisation, "target": c.target, - "admitted": c.admitted, "ceiling": c.ceiling, "total_admitted": c.total, + "admitted": c.admitted, "cost": c.cost, + "ceiling": c.ceiling, "total_admitted": c.total, }), // A minute an application did not appear in is a minute it // admitted nothing, which is a real zero and not a gap. None => { - json!({ "t": t, "utilisation": 0.0, "admitted": 0, "total_admitted": 0 }) + json!({ "t": t, "utilisation": 0.0, "admitted": 0, + "cost": 0.0, "total_admitted": 0 }) } }) .collect(); @@ -423,9 +486,11 @@ pub async fn rollups(State(app): State, Query(q): Query) -> q.target.clone(), ), }; - ok(json!( - h.rollups(&a, &t, q.minutes.unwrap_or(120) as i64).await - )) + let minutes = bounded(q.minutes, 120, MAX_HISTORY_MINUTES) as i64; + ok(json!(h + .rollups(&a, &t, minutes) + .await + .map_err(history_failure)?)) } #[derive(Deserialize)] @@ -443,7 +508,7 @@ pub struct TraceQuery { /// inherit and no `cost_actual` to compare — see the design's §16.5. What is /// kept is the interesting event: the denial. pub async fn traces(State(app): State, Query(q): Query) -> ApiResult { - let limit = q.limit.unwrap_or(100); + let limit = bounded(q.limit, 100, MAX_TRACE_ROWS); let mut out: Vec = app .traces .recent(q.outcome.as_deref(), limit) @@ -454,7 +519,8 @@ pub async fn traces(State(app): State, Query(q): Query) -> A if let Some(h) = app.history.as_ref() { out.extend( h.traces(q.outcome.as_deref(), (limit - out.len()) as i64) - .await, + .await + .map_err(history_failure)?, ); } } @@ -472,9 +538,8 @@ pub struct LimitQuery { /// sees. These are the `brk:` records, which every replica writes and every /// replica can read. pub async fn recent_breaches(State(app): State, Query(q): Query) -> ApiResult { - ok(json!( - crate::breaker::recent(&app.budgets, q.limit.unwrap_or(10) as u32).await - )) + let limit = bounded(q.limit, 10, MAX_BREACH_ROWS) as u32; + ok(json!(crate::breaker::recent(&app.budgets, limit).await?)) } /// One row per `(application, sharedKey)`, read live. @@ -506,8 +571,7 @@ pub async fn shared_budgets(State(app): State) -> ApiResult { let state = app .budgets .read(std::slice::from_ref(&first.key)) - .await - .unwrap_or_default() + .await? .into_iter() .next(); let conflicts: Vec = members @@ -572,7 +636,7 @@ pub async fn app_metrics( let budget_pending: u64 = app .depths .pending_of_group(&app.queen, &s.stage.source, &s.stage.group) - .await + .await? .values() .sum(); waiting_budget += budget_pending; @@ -585,7 +649,8 @@ pub async fn app_metrics( &s.stage.path, now, ) - .await, + .await + .map_err(history_failure)?, ), // Null, not a lifetime average — the same fix as // `/api/overview`, so the two fields of this name finally @@ -610,15 +675,15 @@ pub async fn app_metrics( Some(gr) => app .depths .pending_of_group(&app.queen, q, gr) - .await + .await? .values() .sum(), - None => app.depths.pending(&app.queen, q).await.values().sum(), + None => app.depths.pending(&app.queen, q).await?.values().sum(), }; } let keys: Vec = np.unscoped().map(|b| b.key.clone()).collect(); - let states = app.budgets.read(&keys).await.unwrap_or_default(); + let states = app.budgets.read(&keys).await?; let binding = np .unscoped() .map(|b| { @@ -635,7 +700,7 @@ pub async fn app_metrics( _ => Some(x), }); - let breaker = crate::breaker::held(&app.budgets, np).await; + let breaker = crate::breaker::held(&app.budgets, np).await?; let state = if breaker.is_some() { "breached" } else if waiting_budget > 0 { @@ -710,3 +775,33 @@ pub async fn me( "expires_at": s.exp, })) } + +#[cfg(test)] +mod tests { + use axum::http::StatusCode; + + use super::{bounded, history_failure, MAX_BREACH_ROWS, MAX_HISTORY_MINUTES, MAX_TRACE_ROWS}; + + #[test] + fn console_query_sizes_are_never_zero_or_unrepresentably_large() { + assert_eq!(bounded(None, 120, MAX_HISTORY_MINUTES), 120); + assert_eq!(bounded(Some(0), 120, MAX_HISTORY_MINUTES), 1); + assert_eq!( + bounded(Some(usize::MAX), 120, MAX_HISTORY_MINUTES), + MAX_HISTORY_MINUTES + ); + assert_eq!(bounded(Some(usize::MAX), 100, MAX_TRACE_ROWS), 500); + assert_eq!(bounded(Some(usize::MAX), 10, MAX_BREACH_ROWS), 500); + } + + #[test] + fn a_history_failure_is_not_reported_as_an_empty_success() { + let failure = history_failure("connection refused".into()); + + assert_eq!(failure.0, StatusCode::BAD_GATEWAY); + assert_eq!( + failure.1, + "could not read history: connection refused".to_string() + ); + } +} diff --git a/crates/server/src/api/data.rs b/crates/server/src/api/data.rs index 38b4baf..6797136 100644 --- a/crates/server/src/api/data.rs +++ b/crates/server/src/api/data.rs @@ -23,7 +23,7 @@ use serde_json::{json, Value}; use gate_core::plan::NodePlan; use gate_core::GATE_META; -use crate::api::{find, ok, refuse_if_stopped, resolve, ApiResult, Fail, Shared}; +use crate::api::{find, object_payload, ok, refuse_if_stopped, resolve, ApiResult, Fail, Shared}; use crate::registry::GraphRuntime; #[derive(Debug, Deserialize)] @@ -58,7 +58,7 @@ pub struct PushBody { #[serde(default)] pub cost: Option, #[serde(default)] - pub payload: Value, + pub payload: Option, } pub async fn graph_push( @@ -187,10 +187,7 @@ async fn push_into( } // ---- the envelope. - let mut item = body.payload.clone(); - if !item.is_object() { - item = json!({}); - } + let mut item = object_payload(body.payload.clone())?; { let obj = item.as_object_mut().expect("object"); if !body.op.is_empty() { @@ -223,19 +220,17 @@ async fn push_into( // because a lease that expires charges no retry budget. let cost = gate_core::cost_of(&np.cost, &item) .map_err(|e| Fail(StatusCode::UNPROCESSABLE_ENTITY, e.to_string()))?; - // And a counter keyed on an absent value measures the wrong thing. - for b in np.budgets.iter().filter(|b| b.is_scoped()) { - let path = b.scope_by.as_deref().unwrap_or_default(); - if gate_core::scope_value(&item, path).is_none() { - return Err(Fail( - StatusCode::UNPROCESSABLE_ENTITY, - format!( - "budget `{}` of node `{node}` counts per `{path}` and this push carries none: \ - a counter keyed on an absent value measures the wrong thing.", - b.id - ), - )); - } + // And a counter keyed on an absent value measures the wrong thing. Only an + // applicable budget requires its scope: a `whenOp` that does not match this + // item charges nothing and needs no key. + if let Some((budget, path)) = gate_core::missing_scope(&np.budgets, &item) { + return Err(Fail( + StatusCode::UNPROCESSABLE_ENTITY, + format!( + "budget `{budget}` of node `{node}` counts per `{path}` and this push carries none: \ + a counter keyed on an absent value measures the wrong thing." + ), + )); } // ---- shed load, if and only if the declaration asked for it. @@ -250,7 +245,7 @@ async fn push_into( // for work that has not moved. What it buys is a caller who can back off // instead of filling a queue — 429 with the deadline read off the counter's // own TTL. - if let Some(retry_after) = shed(st, np, cost).await { + if let Some(retry_after) = shed(st, np, &item, cost).await { return Err(Fail( StatusCode::TOO_MANY_REQUESTS, format!( @@ -297,6 +292,9 @@ fn write_path(data: &mut Value, path: &str, value: Value) { return; } segs.remove(0); + if segs.first() == Some(&GATE_META) { + return; + } let last = segs.pop().expect("at least one segment"); let mut cur = data; for s in segs { @@ -342,36 +340,80 @@ fn spread(partitions: Option) -> Option { Some(format!("p{i}")) } -/// `Some(seconds)` when every unscoped counter of this node is already full. +/// `Some(seconds)` when any counter applicable to this item is already full. /// /// Read-only, and best-effort: a broker that will not answer means the push goes /// through and the relay decides, which is the right way round — the door must /// never be the thing that stops work when the limiter itself is fine. -async fn shed(st: &Shared, np: &NodePlan, cost: i64) -> Option { +async fn shed(st: &Shared, np: &NodePlan, item: &Value, cost: i64) -> Option { if !np.ingress_shed { return None; } - let keys: Vec = np.unscoped().map(|b| b.key.clone()).collect(); - if keys.is_empty() { + let applicable = shed_budgets(np, item); + if applicable.is_empty() { return None; } + let keys: Vec = applicable.iter().map(|b| b.key.clone()).collect(); let states = st.budgets.read(&keys).await.ok()?; - let now = crate::now_ms(); - let mut worst: Option = None; - for b in np.unscoped() { - let ceiling = b.max_for(np.widest_share()); - let s = states.iter().find(|s| s.key == b.key)?; - if s.value + cost <= ceiling { - return None; + shed_wait(&applicable, &states, cost, crate::now_ms()) +} + +#[derive(Debug, PartialEq, Eq)] +struct ShedBudget { + key: String, + ceiling: i64, +} + +/// Resolve the exact counters the relay would charge for this payload. +fn shed_budgets(np: &NodePlan, item: &Value) -> Vec { + let op = gate_core::op_of(item); + let mut out = Vec::new(); + for b in &np.budgets { + if b.when_op + .as_ref() + .is_some_and(|patterns| !gate_core::op_matches(patterns, op)) + { + continue; + } + let key = match &b.scope_by { + Some(path) => match gate_core::scope_value(item, path) { + Some(value) => b.key_for(Some(&value)), + None => continue, + }, + None => b.key.clone(), + }; + if out.iter().any(|seen: &ShedBudget| seen.key == key) { + continue; } - let wait = s - .expires_at_ms - .map(|e| ((e - now) as f64 / 1000.0).ceil() as i64) - .unwrap_or(1) - .max(1); - worst = Some(worst.map_or(wait, |w: i64| w.max(wait))); + out.push(ShedBudget { + key, + ceiling: b.max_for(np.widest_share()), + }); } - worst + out +} + +fn shed_wait( + budgets: &[ShedBudget], + states: &[crate::budget::State], + cost: i64, + now: i64, +) -> Option { + budgets + .iter() + .filter_map(|b| { + let s = states.iter().find(|s| s.key == b.key)?; + if s.value <= b.ceiling.saturating_sub(cost) { + return None; + } + let wait = s + .expires_at_ms + .map(|e| ((e - now) as f64 / 1000.0).ceil() as i64) + .unwrap_or(1) + .max(1); + Some(wait) + }) + .max() } // ------------------------------------------------------------------- gone @@ -466,3 +508,102 @@ fn egress_hint(st: &Shared, application: &str, graph: &str, node: &str) -> Strin ), } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn the_v1_cost_shim_cannot_write_the_gate_envelope() { + let mut data = json!({"_gate": {"path": "main"}}); + write_path(&mut data, "payload._gate.path", json!(99)); + assert_eq!(data, json!({"_gate": {"path": "main"}})); + } + + fn budget(id: &str) -> gate_core::CompiledBudget { + gate_core::CompiledBudget { + id: id.into(), + key: format!("key:{id}"), + scope_by: None, + shared_key: None, + when_op: None, + count: 10, + time_ms: 1000, + sub_windows: 1, + count_sub: 10, + window_sub_seconds: 1, + confidence: gate_core::Confidence::Inferred, + } + } + + fn node(budgets: Vec) -> NodePlan { + NodePlan { + name: "n".into(), + budgets, + cost: gate_core::Cost::Fixed(1), + ingress_queue: Some("in".into()), + ingress_owned: true, + ingress_http: true, + ingress_shed: true, + interior_queue: "interior".into(), + egress_queue: Some("out".into()), + egress_group: None, + breaker_key: "breaker".into(), + shares: Default::default(), + } + } + + #[test] + fn one_full_budget_is_enough_to_shed() { + let budgets = vec![ + ShedBudget { + key: "full".into(), + ceiling: 10, + }, + ShedBudget { + key: "room".into(), + ceiling: 10, + }, + ]; + let states = vec![ + crate::budget::State { + key: "full".into(), + value: 10, + expires_at_ms: Some(12_000), + }, + crate::budget::State { + key: "room".into(), + value: 0, + expires_at_ms: Some(20_000), + }, + ]; + + assert_eq!(shed_wait(&budgets, &states, 1, 10_000), Some(2)); + } + + #[test] + fn shed_resolves_when_op_and_scoped_keys_for_this_item() { + let global = budget("global"); + let mut writes = budget("writes"); + writes.when_op = Some(vec!["listing.write".into()]); + let mut customer = budget("customer"); + customer.scope_by = Some("payload.customerId".into()); + let np = node(vec![global, writes, customer]); + + let got = shed_budgets(&np, &json!({ "op": "listing.read", "customerId": "c-7" })); + assert_eq!( + got, + vec![ + ShedBudget { + key: "key:global".into(), + ceiling: 10, + }, + ShedBudget { + key: "key:customer:c-7".into(), + ceiling: 10, + }, + ] + ); + } +} diff --git a/crates/server/src/api/declare.rs b/crates/server/src/api/declare.rs index b3c5204..99fd576 100644 --- a/crates/server/src/api/declare.rs +++ b/crates/server/src/api/declare.rs @@ -7,6 +7,7 @@ #![allow(deprecated)] +use std::collections::BTreeSet; use std::sync::Arc; use axum::extract::{Path, State}; @@ -17,7 +18,7 @@ use serde_json::{json, Value}; use gate_core::{v1, GraphDoc}; -use crate::api::{find, ok, resolve, ApiResult, Fail, Shared}; +use crate::api::{find, ok, resolve, resolve_found, ApiResult, Fail, Shared}; use crate::registry::GraphRuntime; // ------------------------------------------------------------------- reading a body @@ -136,12 +137,12 @@ pub async fn get_graph( Path((application, name)): Path<(String, String)>, ) -> ApiResult { let rt = find(&st, &application, &name)?; - ok(view(&st, &rt).await) + ok(view(&st, &rt).await?) } pub async fn get_graph_default(State(st): State, Path(name): Path) -> ApiResult { let rt = resolve(&st, &name)?; - ok(view(&st, &rt).await) + ok(view(&st, &rt).await?) } pub async fn del_graph( @@ -152,11 +153,16 @@ pub async fn del_graph( } pub async fn del_graph_default(State(st): State, Path(name): Path) -> ApiResult { - let app = match st.registry.resolve(&name) { - crate::registry::Resolved::One(g) => g.doc.application.clone(), - _ => gate_core::default_application(), + // The flat GET and data-plane routes refuse an ambiguous name; DELETE must + // obey the same rule. Falling back to `default` here could remove one of the + // colliding graphs precisely when the caller had not identified which one. + let application = match st.registry.resolve(&name) { + // Preserve idempotent flat deletes: with no live match, the route still + // removes a possibly stored default-application document. + crate::registry::Resolved::None => gate_core::default_application(), + found => resolve_found(&name, found)?.doc.application.clone(), }; - ok(crate::graph::remove(&st, &app, &name).await?) + ok(crate::graph::remove(&st, &application, &name).await?) } pub async fn topology( @@ -242,6 +248,34 @@ async fn do_sync(st: &Shared, application: &str, bodies: Vec) -> ApiResul let mut refused = Vec::new(); let mut declared: Vec = Vec::new(); + // A sync may land on a replica before its registry has reconciled. The + // durable store is therefore part of the inventory to reap, not merely a + // place each local runtime happens to be deleted from. Incomplete is not + // empty: if even one page or row is unreadable, applying submitted targets + // is safe but treating unseen targets as absent is not. + let stored_targets = match crate::store::try_load_all(&st.queen).await { + Ok(stored) if stored.complete => stored + .items + .into_iter() + .filter(|doc| doc.application == application && doc.nodes.len() == 1) + .map(|doc| doc.graph) + .collect::>(), + Ok(_) => { + refused.push(json!({ + "target": "", + "error": "the stored target inventory is incomplete; submitted declarations may apply, but nothing omitted can be removed safely" + })); + Vec::new() + } + Err(e) => { + refused.push(json!({ + "target": "", + "error": format!("the stored target inventory could not be read ({e}); submitted declarations may apply, but nothing omitted can be removed safely") + })); + Vec::new() + } + }; + for body in bodies { // The name comes from the document here, because a sync body is a list // and there is no path segment to pin it from. @@ -271,31 +305,49 @@ async fn do_sync(st: &Shared, application: &str, bodies: Vec) -> ApiResul // Reap, and ONLY inside this application. The flat version of this reaped // everything the cell held, so two teams syncing against one deployment - // would delete each other's graphs — including from the durable store. Done - // AFTER the declares, so a sync that fails half way removes nothing. + // would delete each other's graphs — including from the durable store. + // + // A partial declaration is not an authoritative inventory. One malformed + // body must not turn `ok: false` into a successful deletion of every valid + // target the caller omitted, so a sync that refused anything removes + // nothing. Successfully applied documents stay applied and the caller can + // repair/retry the list without recovering deleted configuration first. let mut removed = Vec::new(); - for rt in st.registry.of_app(application) { - if declared.contains(&rt.doc.graph) { - continue; - } - // A sync reaps what it could have DECLARED, and nothing else. v1 - // exempted graph nodes from a target sync because a target list does not - // name them and reaping one would tear down half a topology; there are - // no node-targets any more, so the same rule is spelt as "a sync of - // targets does not delete a graph". A one-node graph IS a target and is - // fair game. - if rt.plan.nodes.len() > 1 { - continue; - } - let name = rt.doc.graph.clone(); - if let Err(e) = crate::store::forget(&st.queen, application, &name).await { - tracing::warn!(graph = %name, error = %e, "sync: not reaped, the stored document could not be removed"); - refused.push(json!({ "target": name, "error": format!("not reaped: {e}") })); - continue; + if refused.is_empty() { + let mut candidates: BTreeSet = stored_targets.into_iter().collect(); + candidates.extend( + st.registry + .of_app(application) + .into_iter() + .filter(|rt| rt.plan.nodes.len() == 1) + .map(|rt| rt.doc.graph.clone()), + ); + for name in candidates { + if declared.contains(&name) { + continue; + } + // The store's copy decided that a name was a target; the RUNTIME + // decides whether this replica may reap it. A redeclare registers + // before it saves, so a graph that grew nodes and then failed to + // persist reads as a one-node document here and is a multi-node + // graph in the registry — and a sync of targets does not delete a + // graph. Asked before the store write, so a graph that is exempt + // keeps its document too. + let live = st.registry.get(application, &name); + if live.as_ref().is_some_and(|rt| rt.plan.nodes.len() > 1) { + continue; + } + if let Err(e) = crate::store::forget(&st.queen, application, &name).await { + tracing::warn!(graph = %name, error = %e, "sync: not reaped, the stored document could not be removed"); + refused.push(json!({ "target": name, "error": format!("not reaped: {e}") })); + continue; + } + if let Some(rt) = live { + crate::supervisor::stop(&rt).await; + st.registry.remove(application, &name); + } + removed.push(name); } - crate::supervisor::stop(&rt).await; - st.registry.remove(application, &name); - removed.push(name); } ok(json!({ @@ -314,17 +366,57 @@ async fn do_sync(st: &Shared, application: &str, bodies: Vec) -> ApiResul /// The budget bars read the counter itself — value AND `expiresAt` — so the /// console can render the window's remaining time, which v1 could not: its /// mirror was a copy of a state document with no expiry in it. -pub async fn view(st: &Shared, rt: &Arc) -> Value { +pub async fn view(st: &Shared, rt: &Arc) -> queen_mq::Result { let mut nodes = Vec::new(); for (name, np) in &rt.plan.nodes { + // Keep the two owners of backlog separate in the graph view, just as + // the metrics and ETA endpoints do. The first number is work each + // stage has not admitted yet; the second is work Gate has relayed to a + // terminal queue and the application's consumers have not picked up. + let mut waiting_for_budget = 0u64; + for s in rt.stages_of_node(name) { + waiting_for_budget += st + .depths + .pending_of_group(&st.queen, &s.stage.source, &s.stage.group) + .await? + .values() + .sum::(); + } + + let waiting_for_workers = match (&np.egress_queue, &np.egress_group) { + (Some(queue), Some(group)) => st + .depths + .pending_of_group(&st.queen, queue, group) + .await? + .values() + .sum::(), + (Some(queue), None) => st + .depths + .pending(&st.queen, queue) + .await? + .values() + .sum::(), + (None, _) => 0, + }; + let keys: Vec = np.unscoped().map(|b| b.key.clone()).collect(); - let states = st.budgets.read(&keys).await.unwrap_or_default(); - let breaker = crate::breaker::held(&st.budgets, np).await; + let states = st.budgets.read(&keys).await?; + let breaker = crate::breaker::held(&st.budgets, np).await?; let budgets: Vec = np .budgets .iter() - .map(|b| { + .enumerate() + .map(|(index, b)| { + // Enforcement uses the compiled budget, while provenance is + // intentionally documentation-only and remains on the source + // document. The compiler preserves budget order. + let declared = rt + .doc + .nodes + .get(name) + .and_then(|node| node.budgets.get(index)) + .filter(|source| source.id_or(index) == b.id); let s = states.iter().find(|s| s.key == b.key); let ceiling = b.max_for(np.widest_share()); let value = s.map(|s| s.value).unwrap_or(0); @@ -333,12 +425,15 @@ pub async fn view(st: &Shared, rt: &Arc) -> Value { "key": b.key, "scopeBy": b.scope_by, "sharedKey": b.shared_key, + "whenOp": b.when_op, "count": b.count, "timeMs": b.time_ms, "subWindows": b.sub_windows, "countSub": b.count_sub, "windowSubSeconds": b.window_sub_seconds, "confidence": b.confidence, + "source": declared.and_then(|source| source.source.as_ref()), + "asOf": declared.and_then(|source| source.as_of.as_ref()), // A per-key budget has no single counter to report: the // number that matters is the worst live key, and finding it // means enumerating a namespace. `null` says so rather than @@ -367,10 +462,12 @@ pub async fn view(st: &Shared, rt: &Arc) -> Value { "paths": gate_core::plan::paths_through(&rt.plan, name), "shares": np.shares, "budgets": budgets, + "waiting_for_budget": waiting_for_budget, + "waiting_for_workers": waiting_for_workers, "breaker": breaker.map(|b| json!({ "at": b.at, "retryAfterSeconds": b.retry_after_seconds, - "until": b.at + b.retry_after_seconds * 1000, + "until": b.until_ms(), "by": b.by, })), })); @@ -388,7 +485,7 @@ pub async fn view(st: &Shared, rt: &Arc) -> Value { let lag: u64 = st .depths .pending_of_group(&st.queen, &s.stage.source, &s.stage.group) - .await + .await? .values() .sum(); stages.push(json!({ @@ -408,7 +505,7 @@ pub async fn view(st: &Shared, rt: &Arc) -> Value { })); } - json!({ + Ok(json!({ "application": rt.doc.application, "graph": rt.doc.graph, "name": rt.doc.graph, @@ -426,9 +523,12 @@ pub async fn view(st: &Shared, rt: &Arc) -> Value { "hops": gate_core::plan::hop_names(p), })).collect::>(), "spec": rt.doc, - }) + })) } pub async fn view_response(st: &Shared, rt: &Arc) -> impl IntoResponse { - Json(view(st, rt).await) + match view(st, rt).await { + Ok(value) => Json(value).into_response(), + Err(error) => Fail::from(error).into_response(), + } } diff --git a/crates/server/src/api/eta.rs b/crates/server/src/api/eta.rs index e0c0dee..4b70829 100644 --- a/crates/server/src/api/eta.rs +++ b/crates/server/src/api/eta.rs @@ -64,7 +64,7 @@ async fn answer( )); } let path = pick(rt, node, &q)?; - match crate::eta::view(st, rt, node, &path).await { + match crate::eta::view(st, rt, node, &path).await? { Some(v) => ok(v), None => Err(Fail( StatusCode::NOT_FOUND, diff --git a/crates/server/src/api/mod.rs b/crates/server/src/api/mod.rs index b9934c4..970dbd1 100644 --- a/crates/server/src/api/mod.rs +++ b/crates/server/src/api/mod.rs @@ -23,14 +23,14 @@ pub mod reenter; use std::sync::Arc; use std::time::Instant; -use axum::extract::State; +use axum::extract::{DefaultBodyLimit, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post, put}; use axum::{Json, Router}; use parking_lot::RwLock; use queen_mq::Queen; -use serde_json::json; +use serde_json::{json, Value}; use crate::budget::Budgets; use crate::obs::Traces; @@ -42,6 +42,7 @@ pub struct App { pub budgets: Budgets, pub registry: Registry, pub depths: Arc, + pub backlogs: crate::obs::BacklogTrends, pub traces: Arc, pub history: Option>, pub queen_url: String, @@ -75,6 +76,7 @@ impl App { queen, registry: Default::default(), depths: Arc::new(crate::depth::Depths::default()), + backlogs: Default::default(), traces: Arc::new(Traces::default()), history: None, queen_url, @@ -128,7 +130,7 @@ pub fn public_router(app: Shared) -> Router { routes() .route("/api/auth/google/login", get(crate::auth::login)) .route("/api/auth/google/callback", get(crate::auth::callback)) - .route("/api/auth/logout", get(crate::auth::logout)) + .route("/api/auth/logout", post(crate::auth::logout)) .merge(crate::webapp::router()) .layer(axum::middleware::from_fn_with_state( app.clone(), @@ -137,6 +139,21 @@ pub fn public_router(app: Shared) -> Router { .with_state(app) } +/// The body limit the four push routes carry, and only they. +/// +/// A push is a BATCH: one request stands for as many items as the caller managed +/// to group, and the honest bound on it is memory. Everything else on this +/// surface takes a document — a graph declaration, a breaker poke, a console +/// read — and keeps axum's 2 MiB, which no document has ever come near. +/// +/// Applied per route rather than as one `.layer()` on the Router for exactly +/// that reason: a limit on the whole surface would raise the ceiling on +/// endpoints that have no batch to justify it, and this service holds every +/// buffered body in a 512 MiB pod. +fn push_body_limit() -> DefaultBodyLimit { + DefaultBodyLimit::max(crate::knobs::knobs().max_push_body) +} + fn routes() -> Router { Router::new() // ---- graphs: the one document type. @@ -154,11 +171,11 @@ fn routes() -> Router { ) .route( "/v1/apps/:app/graphs/:graph/nodes/:node/push", - post(data::graph_push), + post(data::graph_push).layer(push_body_limit()), ) .route( "/v1/graphs/:graph/nodes/:node/push", - post(data::graph_push_default), + post(data::graph_push_default).layer(push_body_limit()), ) .route( "/v1/apps/:app/graphs/:graph/nodes/:node/eta", @@ -206,11 +223,11 @@ fn routes() -> Router { ) .route( "/v1/apps/:app/targets/:name/lanes/:lane/push", - post(data::target_push), + post(data::target_push).layer(push_body_limit()), ) .route( "/v1/targets/:name/lanes/:lane/push", - post(data::target_push_default), + post(data::target_push_default).layer(push_body_limit()), ) .route("/v1/apps/:app/targets/:name/eta", get(eta::target_eta)) .route("/v1/targets/:name/eta", get(eta::target_eta_default)) @@ -289,6 +306,32 @@ pub fn ok(v: serde_json::Value) -> ApiResult { Ok(Json(v).into_response()) } +/// Gate's HTTP doors add `_gate` metadata to the application payload. Refuse a +/// shape that cannot carry that metadata instead of silently replacing the +/// caller's data with an empty object. +pub(crate) fn object_payload(payload: Option) -> Result { + // An ABSENT payload is not a payload Gate would be erasing: there is nothing + // to lose, and `{"op": "publish", "txn": "t1"}` has always been a legal push + // of an item that carries only its envelope. `serde` cannot tell an absent + // field from an explicit `null`, so the distinction is drawn here, on + // `Option`, rather than on `Value::Null`. + let Some(payload) = payload else { + return Ok(json!({})); + }; + if payload.is_object() { + return Ok(payload); + } + Err(Fail( + StatusCode::UNPROCESSABLE_ENTITY, + format!( + "payload must be a JSON object: Gate must add `{}` metadata without changing the \ + application value. Omit the field entirely to push an item that carries only its \ + envelope.", + gate_core::GATE_META + ), + )) +} + impl From for Fail { fn from(r: crate::graph::Refusal) -> Self { match r { @@ -299,6 +342,15 @@ impl From for Fail { } } +impl From for Fail { + fn from(error: queen_mq::Error) -> Self { + Fail( + StatusCode::BAD_GATEWAY, + format!("could not read live broker state: {error}"), + ) + } +} + /// Find a graph, by application and name. pub fn find( st: &Shared, @@ -315,7 +367,14 @@ pub fn find( /// The one case the server refuses to guess: two applications with a graph of /// one name. Picking either would run somebody else's declaration. pub fn resolve(st: &Shared, name: &str) -> Result, Fail> { - match st.registry.resolve(name) { + resolve_found(name, st.registry.resolve(name)) +} + +pub(crate) fn resolve_found( + name: &str, + found: crate::registry::Resolved, +) -> Result, Fail> { + match found { crate::registry::Resolved::One(g) => Ok(g), crate::registry::Resolved::None => { Err(Fail(StatusCode::NOT_FOUND, format!("no graph `{name}`"))) @@ -351,3 +410,41 @@ pub fn refuse_if_stopped(rt: &Arc) -> Result<(), ), )) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_http_payload_must_be_an_object_instead_of_being_discarded() { + let kept = object_payload(Some(json!({ "kept": true }))) + .ok() + .expect("object refused"); + assert_eq!(kept["kept"], true); + for value in [json!(7), json!("lost"), json!([1, 2])] { + let err = object_payload(Some(value)).expect_err("non-object accepted"); + assert_eq!(err.0, StatusCode::UNPROCESSABLE_ENTITY); + assert!(err.1.contains("must be a JSON object")); + } + } + + /// An omitted `payload` erases nothing, so it is not the shape this refusal + /// is about: `{"op": "publish", "txn": "t1"}` is a push of an item that + /// carries only its envelope, and it answered 200 before this rule existed. + #[test] + fn an_omitted_payload_is_an_empty_object_and_not_a_refusal() { + let absent = object_payload(None) + .ok() + .expect("an absent payload was refused"); + assert_eq!(absent, json!({})); + let body: crate::api::data::PushBody = + serde_json::from_value(json!({ "op": "publish", "txn": "t1" })).expect("body"); + assert_eq!(body.payload, None, "an absent field must not read as null"); + // `serde` maps an explicit `null` onto `None` as well, and that is the + // right reading: both spellings say "this item carries no payload", and + // neither has anything for Gate to erase. + let body: crate::api::data::PushBody = + serde_json::from_value(json!({ "payload": null })).expect("body"); + assert_eq!(body.payload, None); + } +} diff --git a/crates/server/src/api/reenter.rs b/crates/server/src/api/reenter.rs index fd6d5a1..8246c38 100644 --- a/crates/server/src/api/reenter.rs +++ b/crates/server/src/api/reenter.rs @@ -38,14 +38,14 @@ use serde_json::{json, Value}; use gate_core::GATE_META; -use crate::api::{find, ok, refuse_if_stopped, resolve, ApiResult, Fail, Shared}; +use crate::api::{find, object_payload, ok, refuse_if_stopped, resolve, ApiResult, Fail, Shared}; use crate::registry::GraphRuntime; #[derive(Debug, Deserialize)] pub struct ReenterBody { /// The payload as it was popped off the egress queue, `_gate` and all. #[serde(default)] - pub payload: Value, + pub payload: Option, /// The transaction id it arrived with. Required: it is what the re-entry id /// is derived from, and without it two reports of one item would re-enter /// twice. @@ -85,7 +85,15 @@ pub async fn graph_reenter_default( async fn reenter(st: &Shared, rt: &std::sync::Arc, body: ReenterBody) -> ApiResult { refuse_if_stopped(rt)?; - let stamp = body.payload.get(GATE_META); + validate_parent_txn(&body.txn).map_err(|why| { + Fail( + StatusCode::UNPROCESSABLE_ENTITY, + format!("cannot re-enter an item: {why}"), + ) + })?; + + let mut item = object_payload(body.payload.clone())?; + let stamp = item.get(GATE_META); let path = body .path .clone() @@ -143,11 +151,12 @@ async fn reenter(st: &Shared, rt: &std::sync::Arc, body: ReenterBo )); }; - let was = stamp - .and_then(|g| g.get("attempt")) - .and_then(|a| a.as_u64()) - .unwrap_or(0) as u32; - let attempt = body.attempt.unwrap_or(was + 1); + let attempt = reentry_attempt(stamp, body.attempt).map_err(|why| { + Fail( + StatusCode::UNPROCESSABLE_ENTITY, + format!("cannot re-enter an item in `{}`: {why}", rt.key()), + ) + })?; let max = rt.plan.max_attempts; if attempt > max { return Err(Fail( @@ -164,10 +173,6 @@ async fn reenter(st: &Shared, rt: &std::sync::Arc, body: ReenterBo // Restamped at hop 0 of its own path, with the attempt on it. The relay // carries `attempt` forward across every hop, so the next report of this // item counts from here rather than starting again at one. - let mut item = body.payload.clone(); - if !item.is_object() { - item = json!({}); - } { let obj = item.as_object_mut().expect("object"); obj.insert( @@ -216,3 +221,71 @@ async fn reenter(st: &Shared, rt: &std::sync::Arc, body: ReenterBo "pushed": pushed.len(), })) } + +fn validate_parent_txn(txn: &str) -> Result<(), &'static str> { + if txn.trim().is_empty() { + return Err("`txn` must not be empty; it is the identity used to deduplicate reports"); + } + Ok(()) +} + +/// Pick a strictly later attempt without truncating an untrusted JSON number. +/// A caller may skip forward, but it may never reset the attempt carried by the +/// item: doing so would turn a bounded re-entry into a livelock. +fn reentry_attempt(stamp: Option<&Value>, requested: Option) -> Result { + let was = match stamp.and_then(|g| g.get("attempt")) { + None => 0, + Some(raw) => { + let n = raw.as_u64().ok_or_else(|| { + "`_gate.attempt` must be a non-negative integer when it is present".to_string() + })?; + u32::try_from(n) + .map_err(|_| format!("`_gate.attempt` of {n} is too large to be a valid attempt"))? + } + }; + + let attempt = match requested { + Some(n) => n, + None => was + .checked_add(1) + .ok_or_else(|| format!("`_gate.attempt` of {was} cannot be incremented any further"))?, + }; + if attempt <= was { + return Err(format!( + "attempt {attempt} does not advance the payload's existing attempt {was}" + )); + } + Ok(attempt) +} + +#[cfg(test)] +mod tests { + use super::{reentry_attempt, validate_parent_txn}; + use serde_json::json; + + #[test] + fn a_reentry_attempt_must_move_forward() { + let stamp = json!({ "attempt": 2 }); + assert_eq!(reentry_attempt(Some(&stamp), None), Ok(3)); + assert_eq!(reentry_attempt(Some(&stamp), Some(4)), Ok(4)); + assert!(reentry_attempt(Some(&stamp), Some(2)).is_err()); + assert!(reentry_attempt(Some(&stamp), Some(0)).is_err()); + } + + #[test] + fn an_untrusted_attempt_neither_truncates_nor_overflows() { + let too_large = json!({ "attempt": u64::from(u32::MAX) + 1 }); + let at_limit = json!({ "attempt": u32::MAX }); + let malformed = json!({ "attempt": "many" }); + assert!(reentry_attempt(Some(&too_large), None).is_err()); + assert!(reentry_attempt(Some(&at_limit), None).is_err()); + assert!(reentry_attempt(Some(&malformed), None).is_err()); + } + + #[test] + fn a_reentry_needs_a_real_parent_transaction() { + assert!(validate_parent_txn("").is_err()); + assert!(validate_parent_txn(" \n").is_err()); + assert!(validate_parent_txn("parent-42").is_ok()); + } +} diff --git a/crates/server/src/auth.rs b/crates/server/src/auth.rs index 4f511d7..68bb6ab 100644 --- a/crates/server/src/auth.rs +++ b/crates/server/src/auth.rs @@ -7,9 +7,10 @@ //! rule can expose something by accident, and the same router serves both ports //! unchanged. //! -//! Two exemptions, both structural rather than chosen: `/auth/*`, because you -//! cannot require a session in order to obtain one, and the static shell, which -//! has to render the page the sign-in button lives on. +//! Two exemptions, both structural rather than chosen: the two sign-in +//! bootstrap routes, because you cannot require a session in order to obtain +//! one, and the static shell, which has to render the page the sign-in button +//! lives on. Logout is deliberately not a bootstrap route. //! //! The claim checks mirror `queen-proxy`'s `validate_google_claims`, including //! its `hd` OR email-domain form. Diverging would give two products in one house @@ -20,7 +21,7 @@ use std::collections::HashMap; use std::time::{Duration, Instant}; use axum::extract::{Query, State}; -use axum::http::{header, StatusCode}; +use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Redirect, Response}; use jsonwebtoken::{ decode, decode_header, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation, @@ -39,6 +40,10 @@ const HTTP_TIMEOUT: Duration = Duration::from_secs(10); /// The signed `state` only has to survive one round trip through Google. const STATE_TTL_S: i64 = 300; pub const COOKIE: &str = "gate_session"; +const OAUTH_STATE_COOKIE: &str = "gate_oauth_state"; +const OAUTH_LOGIN_PATH: &str = "/api/auth/google/login"; +const OAUTH_CALLBACK_PATH: &str = "/api/auth/google/callback"; +const LOGOUT_PATH: &str = "/api/auth/logout"; #[derive(Clone)] pub struct AuthConfig { @@ -216,23 +221,45 @@ static JWKS: RwLock> = RwLock::new(None); /// Google's keys, cached. On a refresh failure a stale copy is preferable to /// locking everyone out: the keys rotate slowly and an outage at Google should -/// not become an outage here. -async fn jwks(http: &reqwest::Client) -> Result { - if let Some((v, at)) = JWKS.read().as_ref() { - if at.elapsed() < JWKS_TTL { - return Ok(v.clone()); +/// not become an outage here. `force` is used only after a token names a key +/// absent from the otherwise-fresh cache; in that case the stale set is already +/// known to be unusable, so a failed download is returned rather than disguised +/// as a successful refresh. +async fn jwks(http: &reqwest::Client, force: bool) -> Result { + if !force { + if let Some((v, at)) = JWKS.read().as_ref() { + if at.elapsed() < JWKS_TTL { + return Ok(v.clone()); + } } } - match http.get(GOOGLE_JWKS_URL).timeout(HTTP_TIMEOUT).send().await { - Ok(r) => match r.json::().await { - Ok(v) => { - *JWKS.write() = Some((v.clone(), Instant::now())); - Ok(v) - } - Err(e) => stale_or(format!("jwks decode: {e}")), - }, - Err(e) => stale_or(format!("jwks fetch: {e}")), + match download_jwks(http).await { + Ok(v) => { + *JWKS.write() = Some((v.clone(), Instant::now())); + Ok(v) + } + Err(e) if force => Err(e), + Err(e) => stale_or(e), + } +} + +async fn download_jwks(http: &reqwest::Client) -> Result { + let response = http + .get(GOOGLE_JWKS_URL) + .timeout(HTTP_TIMEOUT) + .send() + .await + .map_err(|e| format!("jwks fetch: {e}"))? + .error_for_status() + .map_err(|e| format!("jwks fetch: {e}"))?; + let value = response + .json::() + .await + .map_err(|e| format!("jwks decode: {e}"))?; + if value.get("keys").and_then(Value::as_array).is_none() { + return Err("jwks decode: response has no keys array".into()); } + Ok(value) } fn stale_or(err: String) -> Result { @@ -252,20 +279,8 @@ async fn verify_id_token( ) -> Result { let header = decode_header(id_token).map_err(|e| format!("id_token header: {e}"))?; let kid = header.kid.ok_or("id_token has no kid")?; - let keys = jwks(http).await?; - let (n, e) = keys["keys"] - .as_array() - .and_then(|ks| { - ks.iter() - .find(|k| k["kid"].as_str() == Some(&kid)) - .map(|k| { - ( - k["n"].as_str().unwrap_or("").to_string(), - k["e"].as_str().unwrap_or("").to_string(), - ) - }) - }) - .ok_or("no matching jwks key")?; + let keys = jwks(http, false).await?; + let (n, e) = matching_jwk(keys, &kid, || jwks(http, true)).await?; let key = DecodingKey::from_rsa_components(&n, &e).map_err(|e| format!("jwks key: {e}"))?; let mut v = Validation::new(Algorithm::RS256); v.set_audience(std::slice::from_ref(&cfg.client_id)); @@ -275,6 +290,37 @@ async fn verify_id_token( .map_err(|e| format!("id_token: {e}")) } +fn jwk_components(keys: &Value, kid: &str) -> Option<(String, String)> { + keys["keys"].as_array().and_then(|ks| { + ks.iter() + .filter(|k| k["kid"].as_str() == Some(kid)) + .find_map(|k| { + let n = k["n"].as_str()?.to_string(); + let e = k["e"].as_str()?.to_string(); + Some((n, e)) + }) + }) +} + +/// Select a key from the cache, refreshing once when rotation introduced a +/// `kid` the cached set does not contain. A token with a genuinely unknown key +/// still fails after that one bounded retry. +async fn matching_jwk( + keys: Value, + kid: &str, + refresh: F, +) -> Result<(String, String), String> +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + if let Some(key) = jwk_components(&keys, kid) { + return Ok(key); + } + let refreshed = refresh().await?; + jwk_components(&refreshed, kid).ok_or_else(|| format!("no matching jwks key for `{kid}`")) +} + // ------------------------------------------------------------------ routes pub struct Auth { @@ -314,17 +360,40 @@ fn now() -> i64 { crate::now_ms() / 1000 } -/// A nonce with no dependency on a random crate: the session secret is already -/// the thing whose secrecy everything here rests on, so a keyed hash of it plus -/// a monotonic instant is as unguessable as the secret itself. -fn nonce(secret: &[u8]) -> String { - let t = crate::now_ms(); - let mut h: u64 = 1469598103934665603; - for b in secret.iter().chain(t.to_le_bytes().iter()) { - h ^= *b as u64; - h = h.wrapping_mul(1099511628211); +/// A fresh 128-bit OpenID Connect nonce from the operating system. +/// +/// This value is public in the authorization URL, so deriving it from the +/// session secret with a non-cryptographic hash does not make it unpredictable: +/// FNV-1a is reversible once its timestamp suffix is known. Entropy is the +/// property the nonce needs, not secrecy of its input. +fn nonce() -> Result { + let mut bytes = [0_u8; 16]; + getrandom::fill(&mut bytes)?; + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for b in bytes { + out.push(HEX[usize::from(b >> 4)] as char); + out.push(HEX[usize::from(b & 0x0f)] as char); + } + Ok(out) +} + +/// A post-login destination is always local to this console. The state is +/// signed against tampering, but the caller chooses the value before it is +/// signed, so signature verification alone does not prevent an open redirect. +fn safe_next(candidate: Option<&str>) -> String { + let Some(path) = candidate else { + return "/".into(); + }; + if path.starts_with('/') + && !path.starts_with("//") + && !path.contains('\\') + && !path.chars().any(char::is_control) + { + path.to_string() + } else { + "/".into() } - format!("{h:016x}{:x}", t) } pub async fn login( @@ -334,10 +403,16 @@ pub async fn login( let Some(auth) = app.auth.as_ref() else { return (StatusCode::NOT_FOUND, "sign-in is not configured").into_response(); }; - let n = nonce(&auth.cfg.secret); + let n = match nonce() { + Ok(n) => n, + Err(e) => { + tracing::error!(error = %e, "could not generate OAuth nonce"); + return (StatusCode::INTERNAL_SERVER_ERROR, "could not start sign-in").into_response(); + } + }; let state = match auth.sign(&StateClaims { nonce: n.clone(), - next: q.get("next").cloned().unwrap_or_else(|| "/".into()), + next: safe_next(q.get("next").map(String::as_str)), exp: now() + STATE_TTL_S, }) { Ok(s) => s, @@ -354,12 +429,24 @@ pub async fn login( // returned token is checked and this is not. urlencode(auth.cfg.allowed_domains.first().map(String::as_str).unwrap_or("")), ); - Redirect::to(&url).into_response() + let secure = auth.cfg.public_url.starts_with("https://"); + let mut response = Redirect::to(&url).into_response(); + let cookie = oauth_state_cookie(&state, STATE_TTL_S, secure); + let Ok(cookie) = HeaderValue::from_str(&cookie) else { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "could not bind the OAuth state", + ) + .into_response(); + }; + response.headers_mut().append(header::SET_COOKIE, cookie); + response } pub async fn callback( State(app): State, Query(q): Query>, + headers: HeaderMap, ) -> Response { let Some(auth) = app.auth.as_ref() else { return (StatusCode::NOT_FOUND, "sign-in is not configured").into_response(); @@ -368,6 +455,18 @@ pub async fn callback( return (StatusCode::BAD_REQUEST, "missing code or state").into_response(); }; + // A signature proves that Gate minted the state; it does not prove that + // THIS browser initiated the transaction. Without this binding an attacker + // can complete their own Google login and make a victim visit the callback, + // replacing the victim's session with the attacker's identity (login CSRF). + if cookie_value(&headers, OAUTH_STATE_COOKIE) != Some(state.as_str()) { + return ( + StatusCode::BAD_REQUEST, + "invalid or expired OAuth transaction", + ) + .into_response(); + } + let mut v = Validation::new(Algorithm::HS256); v.set_required_spec_claims(&["exp"]); let st = match decode::(state, &DecodingKey::from_secret(&auth.cfg.secret), &v) { @@ -448,40 +547,58 @@ pub async fn callback( }; let secure = auth.cfg.public_url.starts_with("https://"); + // Validate again after state verification so a state minted by an older + // version cannot retain an unsafe destination through a rolling deploy. + let next = safe_next(Some(&st.next)); let cookie = format!( "{COOKIE}={token}; Path=/; HttpOnly; SameSite=Lax; Max-Age={}{}", 8 * 3600, if secure { "; Secure" } else { "" } ); - ( + let mut response = ( StatusCode::SEE_OTHER, - [(header::SET_COOKIE, cookie), (header::LOCATION, st.next)], + [(header::SET_COOKIE, cookie), (header::LOCATION, next)], ) - .into_response() + .into_response(); + // One transaction, one use. Matching the original Path is required for the + // browser to remove the cookie rather than leaving a replayable binding + // behind for the rest of its five-minute lifetime. + if let Ok(clear) = HeaderValue::from_str(&oauth_state_cookie("", 0, secure)) { + response.headers_mut().append(header::SET_COOKIE, clear); + } + response } -pub async fn logout() -> Response { +pub async fn logout(axum::Json(()): axum::Json<()>) -> Response { ( - StatusCode::SEE_OTHER, - [ - ( - header::SET_COOKIE, - format!("{COOKIE}=; Path=/; HttpOnly; Max-Age=0"), - ), - (header::LOCATION, "/".to_string()), - ], + StatusCode::NO_CONTENT, + [( + header::SET_COOKIE, + format!("{COOKIE}=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0"), + )], ) .into_response() } pub fn session_of(headers: &axum::http::HeaderMap, auth: &Auth) -> Option { + let token = cookie_value(headers, COOKIE)?; + auth.verify_session(token) +} + +fn cookie_value<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { let raw = headers.get(header::COOKIE)?.to_str().ok()?; - let token = raw - .split(';') + raw.split(';') .filter_map(|c| c.trim().split_once('=')) - .find(|(k, _)| *k == COOKIE) - .map(|(_, v)| v)?; - auth.verify_session(token) + .find(|(k, _)| *k == name) + .map(|(_, v)| v) +} + +fn oauth_state_cookie(value: &str, max_age: i64, secure: bool) -> String { + format!( + "{OAUTH_STATE_COOKIE}={value}; Path={OAUTH_CALLBACK_PATH}; HttpOnly; SameSite=Lax; \ + Max-Age={max_age}{}", + if secure { "; Secure" } else { "" } + ) } /// A signed-in identity conjured without Google, for running the console on a @@ -524,14 +641,13 @@ pub async fn require_session( next: axum::middleware::Next, ) -> Response { let path = req.uri().path().to_string(); - let exempt = path.starts_with("/api/auth/") || path == "/health" || is_shell(&path); - if exempt { + if session_exempt(&path) { return next.run(req).await; } // The laptop case: no Google client at all, or one that is configured but // deliberately stood down for local work. if let Some(dev) = dev_identity(app.auth.as_ref().map(|a| &a.cfg)) { - if writes(req.method()) && !is_admin(&dev.email) { + if requires_admin(req.method(), &path) && !is_admin(&dev.email) { return ( StatusCode::FORBIDDEN, "read-only: this account is not in GATE_ADMIN_EMAILS", @@ -552,7 +668,7 @@ pub async fn require_session( // Keyed on the METHOD rather than on a list of paths, so a route // added tomorrow cannot be born unprotected because someone forgot // to enumerate it. - if writes(req.method()) && !is_admin(&s.email) { + if requires_admin(req.method(), &path) && !is_admin(&s.email) { return ( StatusCode::FORBIDDEN, "read-only: this account is not in GATE_ADMIN_EMAILS", @@ -593,6 +709,16 @@ fn writes(m: &axum::http::Method) -> bool { !matches!(*m, axum::http::Method::GET | axum::http::Method::HEAD) } +/// Logout mutates only the caller's own browser cookie. It still requires a +/// valid session, but a read-only operator must be able to end that session. +fn requires_admin(method: &axum::http::Method, path: &str) -> bool { + writes(method) && path != LOGOUT_PATH +} + +fn session_exempt(path: &str) -> bool { + path == OAUTH_LOGIN_PATH || path == OAUTH_CALLBACK_PATH || path == "/health" || is_shell(path) +} + fn is_shell(path: &str) -> bool { path == "/" || path.starts_with("/assets/") || path == "/favicon.ico" } @@ -607,3 +733,137 @@ fn urlencode(s: &str) -> String { }) .collect() } + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use axum::http::{header, HeaderMap, HeaderValue, Method}; + use serde_json::json; + + use super::{ + cookie_value, matching_jwk, nonce, oauth_state_cookie, requires_admin, safe_next, + session_exempt, LOGOUT_PATH, OAUTH_CALLBACK_PATH, OAUTH_LOGIN_PATH, OAUTH_STATE_COOKIE, + }; + + #[tokio::test] + async fn an_unknown_google_key_refreshes_the_cache_once() { + let refreshes = AtomicUsize::new(0); + let key = matching_jwk( + json!({ "keys": [{ "kid": "old", "n": "old-n", "e": "AQAB" }] }), + "new", + || async { + refreshes.fetch_add(1, Ordering::Relaxed); + Ok(json!({ "keys": [{ "kid": "new", "n": "new-n", "e": "AQAB" }] })) + }, + ) + .await + .expect("the rotated key is present after refresh"); + + assert_eq!(key, ("new-n".into(), "AQAB".into())); + assert_eq!(refreshes.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn a_cached_google_key_does_not_refresh() { + let refreshes = AtomicUsize::new(0); + let key = matching_jwk( + json!({ "keys": [{ "kid": "current", "n": "current-n", "e": "AQAB" }] }), + "current", + || async { + refreshes.fetch_add(1, Ordering::Relaxed); + Err("refresh should not run".into()) + }, + ) + .await + .expect("the cached key is enough"); + + assert_eq!(key, ("current-n".into(), "AQAB".into())); + assert_eq!(refreshes.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn a_key_unknown_after_refresh_is_rejected_without_looping() { + let refreshes = AtomicUsize::new(0); + let result = matching_jwk(json!({ "keys": [] }), "missing", || async { + refreshes.fetch_add(1, Ordering::Relaxed); + Ok(json!({ "keys": [] })) + }) + .await; + + assert!(result.is_err()); + assert_eq!(refreshes.load(Ordering::Relaxed), 1); + } + + #[test] + fn oauth_nonces_are_fresh_128_bit_hex_values() { + let first = nonce().expect("the operating system provides randomness"); + let second = nonce().expect("the operating system provides randomness"); + assert_eq!(first.len(), 32); + assert!(first.bytes().all(|b| b.is_ascii_hexdigit())); + assert_ne!(first, second); + } + + #[test] + fn logout_requires_a_session_but_not_an_admin() { + assert!(session_exempt(OAUTH_LOGIN_PATH)); + assert!(session_exempt(OAUTH_CALLBACK_PATH)); + assert!(!session_exempt(LOGOUT_PATH)); + assert!(!session_exempt("/api/auth/future-route")); + + assert!(!requires_admin(&Method::POST, LOGOUT_PATH)); + assert!(requires_admin(&Method::POST, "/v1/apps/a/graphs/g")); + assert!(!requires_admin(&Method::GET, "/api/graphs")); + } + + #[test] + fn oauth_next_accepts_only_local_absolute_paths() { + for path in ["/", "/graphs", "/#/apps/a/graphs/g?path=main"] { + assert_eq!(safe_next(Some(path)), path); + } + for path in [ + "https://evil.example", + "//evil.example/path", + "/\\evil.example/path", + "graphs", + "", + "/graphs\r\nLocation: https://evil.example", + ] { + assert_eq!(safe_next(Some(path)), "/", "accepted {path:?}"); + } + assert_eq!(safe_next(None), "/"); + } + + #[test] + fn oauth_state_is_bound_to_the_browser_that_started_it() { + let mut headers = HeaderMap::new(); + assert_eq!(cookie_value(&headers, OAUTH_STATE_COOKIE), None); + + headers.insert( + header::COOKIE, + HeaderValue::from_static("other=x; gate_oauth_state=signed-state"), + ); + assert_eq!( + cookie_value(&headers, OAUTH_STATE_COOKIE), + Some("signed-state") + ); + assert_ne!( + cookie_value(&headers, OAUTH_STATE_COOKIE), + Some("attacker-state") + ); + } + + #[test] + fn oauth_state_cookie_is_short_lived_scoped_and_secure_in_production() { + let cookie = oauth_state_cookie("signed-state", 300, true); + assert!(cookie.starts_with("gate_oauth_state=signed-state;")); + assert!(cookie.contains("Path=/api/auth/google/callback")); + assert!(cookie.contains("HttpOnly")); + assert!(cookie.contains("SameSite=Lax")); + assert!(cookie.contains("Max-Age=300")); + assert!(cookie.ends_with("; Secure")); + + let cleared = oauth_state_cookie("", 0, true); + assert!(cleared.contains("Max-Age=0")); + } +} diff --git a/crates/server/src/breaker.rs b/crates/server/src/breaker.rs index 4f6e3ed..25a701a 100644 --- a/crates/server/src/breaker.rs +++ b/crates/server/src/breaker.rs @@ -71,11 +71,29 @@ pub struct Record { pub node: String, } +impl Record { + /// The wall-clock deadline, safe even when a broker record was written by + /// a broken or newer producer. Display code must not be able to overflow a + /// request merely by reading shared state. + pub fn until_ms(&self) -> i64 { + self.at + .saturating_add(self.retry_after_seconds.saturating_mul(1000)) + } + + fn valid(&self) -> bool { + self.at >= 0 && (MIN_SECONDS..=MAX_SECONDS).contains(&self.retry_after_seconds) + } +} + +fn decode_record(value: Value) -> Option { + serde_json::from_value(value).ok().filter(Record::valid) +} + /// Trip the breaker on one node. /// -/// The ordering is not arbitrary: **refund first**. After the window is spent -/// the counter is at its ceiling, and a refund arriving then would open a hole -/// in the very window we are about to close. +/// The counter writes and the record are one KV batch, hence one PostgreSQL +/// transaction. A caller must never be told the trip failed while an invisible +/// breaker was nevertheless left holding the node. pub async fn trip( budgets: &Budgets, application: &str, @@ -106,33 +124,21 @@ pub async fn trip( ), })); } - - if let Some(refund) = body.refund_cost.filter(|c| *c > 0) { - // A CREDIT and not a refund: there is no charge of ours to identify, so - // there is no window to prove and `min: 0` is the only available guard. - // It is safe here and nowhere else, because the spend below overwrites - // whatever this credited a few microseconds later. - let charges: Vec = first_by_key(node) - .into_iter() - .map(|b| crate::budget::Charge { - key: b.key.clone(), - max: b.max_for(node.widest_share()), - ttl: b.window_sub_seconds, - delta: refund, - budget_id: b.id.clone(), - }) - .collect(); - budgets.credit(&charges).await; + if keys.len() > gate_core::MAX_BREAKER_COUNTERS { + return Ok(too_wide(node, keys.len())); } // The WIDEST path's ceiling, so no path can slip under it: a path at // share 0.5 refuses itself at half the counter, and writing half would leave - // it admitting. + // it admitting. Replacing the old counter also gives back `refundCost`: + // none of the old window survives this write, and after the breaker expires + // the next charge starts a fresh window at zero. A separate decrement before + // this replacement was a no-op on success and leaked capacity if the trip + // subsequently failed. let spend: Vec<(String, i64)> = first_by_key(node) .into_iter() .map(|b| (b.key.clone(), b.max_for(node.widest_share()))) .collect(); - budgets.spend(&spend, seconds).await?; let rec = Record { at: crate::now_ms(), @@ -142,10 +148,12 @@ pub async fn trip( graph: graph.to_string(), node: node.name.clone(), }; - // Fleet-wide, and that is the point: v1's breach ring was per-replica, and a - // breach seen only by the pod nobody is looking at is a breach nobody sees. + // Fleet-wide and atomic with the counters. v1's breach ring was per-replica, + // and a breach seen only by the pod nobody is looking at is a breach nobody + // sees; a counter spent without this record is the same operationally. budgets - .put_json( + .spend_with_record( + &spend, &node.breaker_key, serde_json::to_value(&rec).unwrap_or(json!({})), seconds, @@ -162,9 +170,9 @@ pub async fn trip( "ok": true, "node": node.name, "retryAfterSeconds": seconds, - "until": rec.at + seconds * 1000, + "until": rec.until_ms(), "keys": keys, - "refunded": body.refund_cost.unwrap_or(0), + "refunded": body.refund_cost.filter(|c| *c > 0).unwrap_or(0), })) } @@ -177,6 +185,9 @@ pub async fn reset(budgets: &Budgets, node: &NodePlan) -> queen_mq::Result = unique_keys(node); + if keys.len() > gate_core::MAX_BREAKER_COUNTERS { + return Ok(too_wide(node, keys.len())); + } keys.push(node.breaker_key.clone()); budgets.clear(&keys).await?; tracing::info!(node = %node.name, "breaker reset"); @@ -207,48 +218,100 @@ fn unique_keys(node: &NodePlan) -> Vec { .collect() } +fn too_wide(node: &NodePlan, keys: usize) -> Value { + json!({ + "ok": false, + "error": format!( + "node `{}` has {keys} distinct node-wide counters; its breaker needs one additional \ + operation for the audit record, but the broker accepts at most 256 operations in one \ + atomic call. Redeclare it with at most {} distinct unscoped counters.", + node.name, + gate_core::MAX_BREAKER_COUNTERS + ), + }) +} + /// Every breaker currently holding a node, fleet-wide. -pub async fn recent(budgets: &Budgets, limit: u32) -> Vec { - match budgets.get_prefix("brk:", limit).await { - Ok(rows) => { - let mut out: Vec = rows - .into_iter() - .filter_map(|r| r.value.and_then(|v| serde_json::from_value(v).ok())) - .collect(); - out.sort_by_key(|r| std::cmp::Reverse(r.at)); - out.iter() - .map(|r| { - json!({ - "at": r.at, - "application": r.application, - "target": format!("{}.{}", r.graph, r.node), - "graph": r.graph, - "node": r.node, - "retryAfterSeconds": r.retry_after_seconds, - "until": r.at + r.retry_after_seconds * 1000, - "by": r.by, - }) - }) - .collect() - } - Err(e) => { - tracing::warn!(error = %e, "could not read the breaker records"); - Vec::new() - } - } +pub async fn recent(budgets: &Budgets, limit: u32) -> queen_mq::Result> { + let rows = budgets.get_prefix("brk:", limit).await?; + let mut out: Vec = rows + .into_iter() + .filter_map(|r| r.value.and_then(decode_record)) + .collect(); + out.sort_by_key(|r| std::cmp::Reverse(r.at)); + Ok(out + .iter() + .map(|r| { + json!({ + "at": r.at, + "application": r.application, + "target": format!("{}.{}", r.graph, r.node), + "graph": r.graph, + "node": r.node, + "retryAfterSeconds": r.retry_after_seconds, + "until": r.until_ms(), + "by": r.by, + }) + }) + .collect()) } /// One node's breaker record, if it is currently held. /// /// The record's own TTL is the answer: a key that has expired is a breaker that /// has lifted, and there is nothing to sweep and nothing to clear. -pub async fn held(budgets: &Budgets, node: &NodePlan) -> Option { +pub async fn held(budgets: &Budgets, node: &NodePlan) -> queen_mq::Result> { let rows = budgets .get_raw(std::slice::from_ref(&node.breaker_key)) - .await - .ok()?; - rows.into_iter() + .await?; + Ok(rows + .into_iter() .next() .and_then(|r| r.value) - .and_then(|v| serde_json::from_value(v).ok()) + .and_then(decode_record)) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{decode_record, Record, MAX_SECONDS}; + + #[test] + fn a_breaker_deadline_saturates_instead_of_overflowing() { + let record = Record { + at: i64::MAX - 500, + retry_after_seconds: 1, + by: None, + application: "a".into(), + graph: "g".into(), + node: "n".into(), + }; + assert_eq!(record.until_ms(), i64::MAX); + } + + #[test] + fn malformed_shared_breaker_records_are_not_reported_as_live() { + let record = |seconds| { + json!({ + "at": 1, + "retryAfterSeconds": seconds, + "application": "a", + "graph": "g", + "node": "n" + }) + }; + + assert!(decode_record(record(1)).is_some()); + assert!(decode_record(record(0)).is_none()); + assert!(decode_record(record(MAX_SECONDS + 1)).is_none()); + assert!(decode_record(json!({ + "at": -1, + "retryAfterSeconds": 1, + "application": "a", + "graph": "g", + "node": "n" + })) + .is_none()); + } } diff --git a/crates/server/src/budget.rs b/crates/server/src/budget.rs index fb2f43f..5c07a43 100644 --- a/crates/server/src/budget.rs +++ b/crates/server/src/budget.rs @@ -38,6 +38,7 @@ //! items/s the key sees 170 incr/s against that 33k/s ceiling: two orders of //! magnitude of headroom. +use std::collections::HashSet; use std::sync::Arc; use queen_mq::{Expiry, KvOperation, Queen, Result}; @@ -68,11 +69,89 @@ pub struct Charge { pub struct State { pub key: String, pub value: i64, - /// Epoch millis. `None` where the key was absent — which reads as *retry - /// now*, not *wait for ever*. + /// Epoch millis, and `None` reads as *retry now* rather than *wait for + /// ever*: a batch response that omitted its trailing read, or a row whose + /// expiry the broker did not render in a shape `parse_instant` knows. Only + /// the deadline degrades, which is not worth failing a charge over — see + /// `decode_state`. pub expires_at_ms: Option, } +fn decode_state(row: &queen_mq::KvRow) -> Result { + let value = row + .value + .as_ref() + .and_then(|value| value.as_i64()) + .ok_or_else(|| { + queen_mq::Error::Decode(format!( + "budget counter `{}` is present but is not an integer", + row.key + )) + })?; + // An expiry we cannot read costs the PARK DEADLINE and nothing else: the + // caller degrades to "retry now", which is what an omitted read has always + // meant here. Refusing the whole decode instead would fail the CHARGE, and a + // failed charge is not a refusal — the relay reads it as "the batch did not + // happen", releases the claim and is handed the identical batch back. One + // row with a missing or unrenderable expiry would then stop every path on + // that counter for ever, at twice the write volume, admitting nothing. + // + // The VALUE above is a different case and stays an error: a counter that is + // not an integer cannot be reasoned about at all. + let expires_at_ms = match row.expires_at.as_deref() { + None => { + tracing::warn!( + key = %row.key, + "budget: this counter is present without an expiry, so its wait deadline \ + degrades to `retry now`. A counter row is created by `incr` with a TTL; one \ + without a TTL never rotates and should be deleted" + ); + None + } + Some(raw) => match parse_instant(raw) { + Some(ms) => Some(ms), + None => { + tracing::warn!( + key = %row.key, expires_at = %raw, + "budget: this counter's expiry cannot be read, so its wait deadline \ + degrades to `retry now`" + ); + None + } + }, + }; + Ok(State { + key: row.key.clone(), + value, + expires_at_ms, + }) +} + +fn decode_states<'a>( + rows: &[queen_mq::KvRow], + expected: impl IntoIterator, +) -> Result> { + let expected: HashSet<&str> = expected.into_iter().collect(); + let mut seen: HashSet<&str> = HashSet::with_capacity(rows.len()); + let mut states = Vec::with_capacity(rows.len()); + for row in rows { + if !expected.contains(row.key.as_str()) { + return Err(queen_mq::Error::Decode(format!( + "budget read returned unexpected key `{}`", + row.key + ))); + } + if !seen.insert(row.key.as_str()) { + return Err(queen_mq::Error::Decode(format!( + "budget read returned key `{}` more than once", + row.key + ))); + } + states.push(decode_state(row)?); + } + Ok(states) +} + /// One charge attempt, index-aligned to the charges that produced it. #[derive(Debug, Clone, Default)] pub struct Attempt { @@ -313,11 +392,26 @@ impl Budgets { // the same numbers, minus the expiry — so the prefix arithmetic // still works and only the park deadline degrades to "retry now". match results.last().and_then(|r| r.rows.as_ref()) { - Some(rows) => states.extend(rows.iter().map(|r| State { - key: r.key.clone(), - value: r.value.as_ref().and_then(|v| v.as_i64()).unwrap_or(0), - expires_at_ms: r.expires_at.as_deref().and_then(parse_instant), - })), + Some(rows) => { + let decoded = decode_states(rows, chunk.iter().map(|c| c.key.as_str())); + match decoded { + Ok(decoded) => states.extend(decoded), + Err(error) => { + // The writes in this chunk may already have landed. + // A malformed read cannot return their Attempt to + // the relay, so refund every applied charge whose + // post-value proves which window it reached. + let done = (n * MAX_WRITES_PER_CALL + chunk.len()).min(charges.len()); + let attempt = Attempt { + applied: applied.clone(), + post: post.clone(), + states: Vec::new(), + }; + self.refund(&attempt.refunds(&charges[..done])).await; + return Err(error); + } + } + } None => states.extend(chunk.iter().enumerate().map(|(i, c)| { State { key: c.key.clone(), @@ -426,37 +520,6 @@ impl Budgets { } } - /// Credit a counter that this process never charged — the breaker giving - /// back the token a reporter spent on the call a vendor refused. - /// - /// `min: 0` and nothing else, because there is no charge of ours to identify - /// and therefore no window to prove. It is safe where [`Budgets::refund`] is - /// not because the only caller spends the whole window immediately - /// afterwards, which overwrites whatever this credited. - pub async fn credit(&self, charges: &[Charge]) { - let mut ops = Vec::with_capacity(charges.len()); - for c in charges { - match self - .queen - .kv() - .incr(&self.ns, &c.key, -c.delta, Expiry::seconds(c.ttl.max(1))) - .min(0) - .operation() - { - Ok(op) => ops.push(op), - Err(e) => { - tracing::warn!(key = %c.key, error = %e, "budget: could not stage a credit") - } - } - } - if ops.is_empty() { - return; - } - if let Err(e) = self.queen.kv().batch(ops).await { - tracing::warn!(error = %e, keys = charges.len(), "budget: the credit call failed"); - } - } - /// Read counters without touching them. The ETA, the console and the /// breaker's report; never the hot path. pub async fn read(&self, keys: &[String]) -> Result> { @@ -464,26 +527,28 @@ impl Budgets { return Ok(Vec::new()); } let res = self.queen.kv().get_many(&self.ns, keys.to_vec()).await?; - Ok(res - .rows - .unwrap_or_default() - .iter() - .map(|r| State { - key: r.key.clone(), - value: r.value.as_ref().and_then(|v| v.as_i64()).unwrap_or(0), - expires_at_ms: r.expires_at.as_deref().and_then(parse_instant), - }) - .collect()) + decode_states( + res.rows.as_deref().unwrap_or_default(), + keys.iter().map(String::as_str), + ) } - /// Spend a window outright — the breaker. + /// Spend a window outright and publish the breaker record atomically. /// /// `put`'s TTL is **not** create-only (only `incr`'s is), so this rewrites /// both the value and the expiry in one call. That is what makes every /// parked consumer's `expiresAt` the vendor's own `Retry-After` deadline - /// without anybody being told it. - pub async fn spend(&self, keys: &[(String, i64)], ttl_seconds: i64) -> Result<()> { - let mut ops = Vec::with_capacity(keys.len()); + /// without anybody being told it. The record belongs in the same KV batch: + /// `kv_apply_v1` applies a batch in one database transaction, so either the + /// node is both held and visible, or neither write happened. + pub async fn spend_with_record( + &self, + keys: &[(String, i64)], + record_key: &str, + record: serde_json::Value, + ttl_seconds: i64, + ) -> Result<()> { + let mut ops = Vec::with_capacity(keys.len() + 1); for (key, value) in keys { ops.push( self.queen @@ -497,6 +562,17 @@ impl Budgets { .operation()?, ); } + ops.push( + self.queen + .kv() + .put( + &self.ns, + record_key, + record, + Expiry::seconds(ttl_seconds.max(1)), + ) + .operation()?, + ); self.queen.kv().batch(ops).await?; Ok(()) } @@ -526,20 +602,6 @@ impl Budgets { Ok(res.rows.unwrap_or_default()) } - pub async fn put_json( - &self, - key: &str, - value: serde_json::Value, - ttl_seconds: i64, - ) -> Result<()> { - self.queen - .kv() - .put(&self.ns, key, value, Expiry::seconds(ttl_seconds.max(1))) - .send() - .await?; - Ok(()) - } - pub async fn get_prefix(&self, prefix: &str, limit: u32) -> Result> { let res = self .queen @@ -580,6 +642,50 @@ pub fn parse_instant(s: &str) -> Option { mod tests { use super::*; + fn row(value: serde_json::Value, expires_at: Option<&str>) -> queen_mq::KvRow { + queen_mq::KvRow { + key: "budget:a:g:n:b".into(), + value: Some(value), + version: 1, + expires_at: expires_at.map(str::to_string), + updated_at: None, + } + } + + #[test] + fn a_present_budget_counter_requires_an_integer_and_an_expiry() { + let state = + decode_state(&row(json!(7), Some("2025-08-21T08:00:00Z"))).expect("valid counter row"); + assert_eq!(state.value, 7); + assert_eq!(state.expires_at_ms, Some(1_755_763_200_000)); + + assert!(decode_state(&row(json!("7"), Some("2025-08-21T08:00:00Z"))).is_err()); + // The deadline, and only the deadline, degrades. + assert_eq!( + decode_state(&row(json!(7), None)) + .expect("readable") + .expires_at_ms, + None + ); + assert_eq!( + decode_state(&row(json!(7), Some("not-a-time"))) + .expect("readable") + .expires_at_ms, + None + ); + } + + #[test] + fn a_budget_read_accepts_only_the_keys_it_asked_for_once() { + let valid = row(json!(7), Some("2025-08-21T08:00:00Z")); + assert!(decode_states(std::slice::from_ref(&valid), [valid.key.as_str()]).is_ok()); + assert!(decode_states(&[valid.clone(), valid.clone()], [valid.key.as_str()]).is_err()); + + let mut unexpected = valid; + unexpected.key = "budget:somebody-else".into(); + assert!(decode_states(&[unexpected], ["budget:a:g:n:b"]).is_err()); + } + #[test] fn a_broker_timestamp_parses_in_every_shape_it_arrives_in() { let want = 1_755_763_200_000i64; diff --git a/crates/server/src/depth.rs b/crates/server/src/depth.rs index 67859cb..cc4a21d 100644 --- a/crates/server/src/depth.rs +++ b/crates/server/src/depth.rs @@ -25,44 +25,60 @@ use queen_mq::Queen; /// interval and long enough to collapse a burst of them. const TTL: Duration = Duration::from_secs(2); +type Depth = HashMap; + +#[derive(Clone)] +enum Cached { + Value(Depth), + Unavailable(String), +} + #[derive(Default)] pub struct Depths { - // One map, keyed by queue (or `queue\u{1f}group`), holding the last answer - // and when it was given. Spelling the pair out as a type alias would name - // something nobody says out loud. - #[allow(clippy::type_complexity)] - cache: RwLock, Instant)>>, + // One map, keyed by queue (or `queue\u{1f}group`), holding either the last + // answer or a recent failure. Failures are cached too: reporting an outage + // honestly must not turn a page with a dozen graphs into an admin-API retry + // storm. + cache: RwLock>, } impl Depths { - /// Pending count per partition of one queue. An absent queue is zero rather - /// than an error: a target declared a moment ago has no queue yet, and that - /// is a true statement about its backlog. - pub async fn pending(&self, queen: &Queen, queue: &str) -> HashMap { - if let Some((v, at)) = self.cache.read().get(queue) { - if at.elapsed() < TTL { - return v.clone(); - } + fn cached(&self, key: &str) -> Option> { + let cache = self.cache.read(); + let (entry, at) = cache.get(key)?; + if at.elapsed() >= TTL { + return None; } - match self.try_pending_now(queen, queue).await { - Some(v) => v, - // The broker did not answer. Serve the last thing it DID say rather than a - // zero, and stamp it so an outage costs one round trip per TTL instead of one - // per caller: a console polling every few seconds across a dozen targets would - // otherwise hammer an admin API that is already unhappy. - None => { - let stale = self - .cache - .read() - .get(queue) - .map(|(v, _)| v.clone()) - .unwrap_or_default(); - self.cache - .write() - .insert(queue.to_string(), (stale.clone(), Instant::now())); - stale - } + Some(match entry { + Cached::Value(value) => Ok(value.clone()), + Cached::Unavailable(error) => Err(queen_mq::Error::Network(format!( + "cached depth read failure: {error}" + ))), + }) + } + + fn remember_value(&self, key: &str, value: &Depth) { + self.cache.write().insert( + key.to_string(), + (Cached::Value(value.clone()), Instant::now()), + ); + } + + fn remember_failure(&self, key: &str, error: &queen_mq::Error) { + self.cache.write().insert( + key.to_string(), + (Cached::Unavailable(error.to_string()), Instant::now()), + ); + } + + /// Pending count per partition of one queue. A confirmed absent queue is + /// zero; a broker that did not answer is an error. Both successful answers + /// and failures are held briefly. + pub async fn pending(&self, queen: &Queen, queue: &str) -> queen_mq::Result { + if let Some(cached) = self.cached(queue) { + return cached; } + self.try_pending_now(queen, queue).await } /// The same read with the cache skipped, and the answer left in it. @@ -71,8 +87,8 @@ impl Depths { /// every loop — a couple of hundred milliseconds. A two-second-old depth would /// let it overshoot the window by everything it forwarded in the meantime, /// which is the one number the window exists to hold down. - pub async fn pending_now(&self, queen: &Queen, queue: &str) -> HashMap { - self.try_pending_now(queen, queue).await.unwrap_or_default() + pub async fn pending_now(&self, queen: &Queen, queue: &str) -> queen_mq::Result { + self.try_pending_now(queen, queue).await } /// The same read, with the failure kept. @@ -82,47 +98,34 @@ impl Depths { /// failed depth as zero would let it forward a full window on every loop, for as /// long as the admin API is unhappy, and the queue it is supposed to keep shallow /// would grow without a bound. - pub async fn try_pending_now( - &self, - queen: &Queen, - queue: &str, - ) -> Option> { - let mut out = HashMap::new(); - let mut answered = false; - + pub async fn try_pending_now(&self, queen: &Queen, queue: &str) -> queen_mq::Result { // The depth route first (broker >= 1.0.4): watermark arithmetic only — // measured at ~1ms on a gate-sized queue, against two console-grade // queries for the detail below. No group, on purpose: queue-level // pending under the worst-cursor precedence is exactly what the old // detail reported, so the relay's window bound does not move. - match queen.admin().queue_depth(queue, None).await { - Ok(v) => { - answered = true; - out = depth_route(&v); - } + let result = match queen.admin().queue_depth(queue, None).await { + Ok(v) => depth_route(&v), // A 404 here is BOTH "this broker predates the route" and "no such // queue", and they cannot be told apart. The queue detail below // answers both the same way this function always has — it exists // on every broker version, and it 404s a missing queue too — so // one fallback covers both, at the old price only on old brokers. - Err(e) if e.status() == Some(404) => { - if let Ok(v) = queen.admin().queue(queue).await { - answered = true; - out = queue_detail(&v); - } - } - Err(_) => {} - } - if !answered { - // An absent queue answers, and answers zero — a target declared a moment - // ago has no queue yet, and that is a true statement about its backlog. - // This is the other case: the broker did not answer at all. - return None; + Err(e) if e.status() == Some(404) => match queen.admin().queue(queue).await { + Ok(v) => queue_detail(&v), + // The detail route exists on every supported broker, so its + // own 404 confirms that the queue is absent. That is a known + // empty backlog, unlike a transport or server failure. + Err(e) if e.status() == Some(404) => Ok(HashMap::new()), + Err(e) => Err(e), + }, + Err(e) => Err(e), + }; + match &result { + Ok(value) => self.remember_value(queue, value), + Err(error) => self.remember_failure(queue, error), } - self.cache - .write() - .insert(queue.to_string(), (out.clone(), Instant::now())); - Some(out) + result } /// The same read as [`Self::pending_of_group`], with the cache skipped and the @@ -140,16 +143,10 @@ impl Depths { queen: &Queen, queue: &str, group: &str, - ) -> Option> { + ) -> queen_mq::Result { let key = format!("{queue}\u{1f}{group}"); - match queen.admin().queue_depth(queue, Some(group)).await { - Ok(v) => { - let out = depth_route(&v); - self.cache - .write() - .insert(key, (out.clone(), Instant::now())); - Some(out) - } + let result = match queen.admin().queue_depth(queue, Some(group)).await { + Ok(v) => depth_route(&v), // No fallback to the queue-level number, and this one is measured. The // queue-level pending is not this group's backlog on every broker: on // 1.0.3 an admitted queue that a second consumer group had read to the @@ -157,10 +154,17 @@ impl Depths { // A caller that BOUNDS work on this — the relay decides which partitions // to poll — would have stopped polling a queue that was full, which is // not a slower relay but a stopped graph. So an answer that is not this - // group's own is no answer: `None`, and the caller does what it did - // before it could ask. - Err(_) => None, + // group's own is no answer, and the caller decides what it can safely + // do without it. + Err(e) => Err(e), + }; + // This exact read deliberately has no legacy fallback. Do not let its + // 404 poison the normal reader's cache: that reader can still answer + // safely from the queue-level route on an older broker. + if let Ok(value) = &result { + self.remember_value(&key, value); } + result } /// The same read, scoped to ONE consumer group's own backlog. @@ -182,23 +186,15 @@ impl Depths { queen: &Queen, queue: &str, group: &str, - ) -> HashMap { + ) -> queen_mq::Result { // The group belongs in the cache key: two answers about one queue that // mean different things must not serve each other's entry. let key = format!("{queue}\u{1f}{group}"); - if let Some((v, at)) = self.cache.read().get(&key) { - if at.elapsed() < TTL { - return v.clone(); - } + if let Some(cached) = self.cached(&key) { + return cached; } - match queen.admin().queue_depth(queue, Some(group)).await { - Ok(v) => { - let out = depth_route(&v); - self.cache - .write() - .insert(key, (out.clone(), Instant::now())); - out - } + let result = match queen.admin().queue_depth(queue, Some(group)).await { + Ok(v) => depth_route(&v), Err(e) if e.status() == Some(404) => { let out = self.pending(queen, queue).await; // Stamped under the GROUP key as well, and not only under the @@ -209,58 +205,109 @@ impl Depths { // one round trip per caller, which is the thing the TTL exists // to stop. Re-probed once per TTL, so an upgrade or a first push // is noticed two seconds later. - self.cache - .write() - .insert(key, (out.clone(), Instant::now())); out } - // The broker did not answer. Serve the last thing it DID say, stamped, - // for the reason `pending` does it: an outage costs one round trip per - // TTL rather than one per caller. - Err(_) => { - let stale = self - .cache - .read() - .get(&key) - .map(|(v, _)| v.clone()) - .unwrap_or_default(); - self.cache - .write() - .insert(key, (stale.clone(), Instant::now())); - stale - } + Err(e) => Err(e), + }; + match &result { + Ok(value) => self.remember_value(&key, value), + Err(error) => self.remember_failure(&key, error), } + result } } /// `{partitions: [{partition, pending}]}` — what the depth route answers, with /// or without a group. -fn depth_route(v: &serde_json::Value) -> HashMap { +fn depth_route(v: &serde_json::Value) -> queen_mq::Result { let mut out = HashMap::new(); - if let Some(parts) = v.get("partitions").and_then(|p| p.as_array()) { - for p in parts { - let name = p.get("partition").and_then(|n| n.as_str()).unwrap_or(""); - let pending = p.get("pending").and_then(|n| n.as_u64()).unwrap_or(0); - out.insert(name.to_string(), pending); + let parts = v + .get("partitions") + .and_then(|p| p.as_array()) + .ok_or_else(|| queen_mq::Error::Decode("depth response has no partitions array".into()))?; + for (index, p) in parts.iter().enumerate() { + let name = p.get("partition").and_then(|n| n.as_str()).ok_or_else(|| { + queen_mq::Error::Decode(format!( + "depth response partition {index} has no string partition" + )) + })?; + let pending = p.get("pending").and_then(|n| n.as_u64()).ok_or_else(|| { + queen_mq::Error::Decode(format!( + "depth response partition {index} has no unsigned pending" + )) + })?; + if out.insert(name.to_string(), pending).is_some() { + return Err(queen_mq::Error::Decode(format!( + "depth response repeats partition {name}" + ))); } } - out + Ok(out) } /// `{partitions: [{name, stats: {pending}}]}` — the console-grade queue detail, /// which every broker version has and which knows nothing about groups. -fn queue_detail(v: &serde_json::Value) -> HashMap { +fn queue_detail(v: &serde_json::Value) -> queen_mq::Result { let mut out = HashMap::new(); - if let Some(parts) = v.get("partitions").and_then(|p| p.as_array()) { - for p in parts { - let name = p.get("name").and_then(|n| n.as_str()).unwrap_or(""); - let pending = p - .get("stats") - .and_then(|s| s.get("pending")) - .and_then(|n| n.as_u64()) - .unwrap_or(0); - out.insert(name.to_string(), pending); + let parts = v + .get("partitions") + .and_then(|p| p.as_array()) + .ok_or_else(|| queen_mq::Error::Decode("queue detail has no partitions array".into()))?; + for (index, p) in parts.iter().enumerate() { + let name = p.get("name").and_then(|n| n.as_str()).ok_or_else(|| { + queen_mq::Error::Decode(format!("queue detail partition {index} has no string name")) + })?; + let pending = p + .get("stats") + .and_then(|s| s.get("pending")) + .and_then(|n| n.as_u64()) + .ok_or_else(|| { + queen_mq::Error::Decode(format!( + "queue detail partition {index} has no unsigned stats.pending" + )) + })?; + if out.insert(name.to_string(), pending).is_some() { + return Err(queen_mq::Error::Decode(format!( + "queue detail repeats partition {name}" + ))); } } - out + Ok(out) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{depth_route, queue_detail}; + + #[test] + fn depth_route_requires_every_value_it_reports() { + let parsed = depth_route(&json!({ + "partitions": [ + { "partition": "p0", "pending": 3 }, + { "partition": "p1", "pending": 0 } + ] + })) + .expect("valid depth response"); + assert_eq!(parsed.get("p0"), Some(&3)); + assert_eq!(parsed.get("p1"), Some(&0)); + + assert!(depth_route(&json!({ "partitions": [{ "partition": "p0" }] })).is_err()); + assert!(depth_route(&json!({})).is_err()); + } + + #[test] + fn legacy_queue_detail_does_not_turn_schema_drift_into_zero() { + let parsed = queue_detail(&json!({ + "partitions": [{ "name": "p0", "stats": { "pending": 7 } }] + })) + .expect("valid queue detail"); + assert_eq!(parsed.get("p0"), Some(&7)); + + assert!(queue_detail(&json!({ + "partitions": [{ "name": "p0", "stats": {} }] + })) + .is_err()); + } } diff --git a/crates/server/src/eta.rs b/crates/server/src/eta.rs index 79cf24d..f4f02af 100644 --- a/crates/server/src/eta.rs +++ b/crates/server/src/eta.rs @@ -67,7 +67,7 @@ pub fn admits( now_ms: i64, ) -> Schedule { let window_seconds = window_seconds.max(1); - let resets_at = now_ms + resets_in_ms.max(0); + let resets_at = now_ms.saturating_add(resets_in_ms.max(0)); // A cap that cannot admit anything never will — no schedule refills it — so // "we cannot say" is the only answer that is not a lie. @@ -89,8 +89,11 @@ pub fn admits( // capful after the first. This is what makes "nothing until it rotates, then // 150 per second" expressible at all. let windows_after_this = ((need / cap as f64).ceil() as i64 - 1).max(0); + // Multiply after widening to f64. Both operands are valid i64 values, but + // a very deep backlog can span enough windows for their integer product to + // overflow before it ever reaches this display-only estimate. let seconds = - resets_in_ms.max(0) as f64 / 1000.0 + (windows_after_this * window_seconds) as f64; + resets_in_ms.max(0) as f64 / 1000.0 + windows_after_this as f64 * window_seconds as f64; Schedule { seconds: Some(seconds), resets_at, @@ -98,9 +101,18 @@ pub fn admits( } /// The answer, for one path through one node. -pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) -> Option { - let stage = rt.plan.stage(path, node)?; - let np = rt.plan.node(node)?; +pub async fn view( + app: &Shared, + rt: &Arc, + node: &str, + path: &str, +) -> queen_mq::Result> { + let Some(stage) = rt.plan.stage(path, node) else { + return Ok(None); + }; + let Some(np) = rt.plan.node(node) else { + return Ok(None); + }; let now = crate::now_ms(); // ---- position. @@ -111,7 +123,7 @@ pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) let waiting_for_budget: u64 = app .depths .pending_of_group(&app.queen, &stage.source, &stage.group) - .await + .await? .values() .sum(); @@ -125,7 +137,7 @@ pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) worker_group_known = true; app.depths .pending_of_group(&app.queen, q, g) - .await + .await? .values() .sum() } @@ -133,7 +145,7 @@ pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) // it is at or above the group being asked about: it can only make // the answer later, never earlier, which is the safe direction for a // bound. - None => app.depths.pending(&app.queen, q).await.values().sum(), + None => app.depths.pending(&app.queen, q).await?.values().sum(), }; } @@ -143,34 +155,15 @@ pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) // A breaker holding the node is the one caveat that explains the whole // answer rather than qualifying it: the window is spent on purpose and the // long `etaSeconds` below is a vendor's `Retry-After`, not a backlog. - let held = crate::breaker::held(&app.budgets, np).await; + let held = crate::breaker::held(&app.budgets, np).await?; let cost_per_item = measured.unwrap_or(np.cost.default_value() as f64); let want = waiting_for_budget as f64 * cost_per_item; // ---- rate. - let keys: Vec = np.unscoped().map(|b| b.key.clone()).collect(); - let states = app.budgets.read(&keys).await.unwrap_or_default(); + let keys: Vec = np.node_wide_rates().map(|b| b.key.clone()).collect(); + let states = app.budgets.read(&keys).await?; - let mut bound: Option<(&CompiledBudget, Schedule)> = None; - for b in np.unscoped() { - let s = states.iter().find(|s| s.key == b.key); - let sched = admits( - b.max_for(stage.share), - b.window_sub_seconds, - s.map(|s| s.value).unwrap_or(0), - s.and_then(|s| s.expires_at_ms) - .map(|e| e - now) - .unwrap_or(0), - want, - now, - ); - // The slowest binds, and "never" beats every number. - let key = |x: &Schedule| x.seconds.unwrap_or(f64::INFINITY); - bound = match bound { - Some(a) if key(&a.1) >= key(&sched) => Some(a), - _ => Some((b, sched)), - }; - } + let bound = binding_schedule(np, stage.share, &states, want, now); let (bound_by, eta_seconds, resets_at) = match bound { Some((b, s)) => ( @@ -187,7 +180,7 @@ pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) "waiting-workers" }; - Some(json!({ + Ok(Some(json!({ "at": now, "application": rt.doc.application, "graph": rt.doc.graph, @@ -203,7 +196,40 @@ pub async fn view(app: &Shared, rt: &Arc, node: &str, path: &str) "waitingForBudget": waiting_for_budget, "waitingForWorkers": waiting_for_workers, "assumes": assumes(rt, np, stage, measured, cost_per_item, worker_group_known, held.as_ref()), - })) + }))) +} + +/// The slowest schedule every queued item is guaranteed to meet. A conditional +/// budget may delay selected operations, but applying it to the whole queue +/// would turn an unknown mix into a confidently late (and false) bound. +fn binding_schedule<'a>( + np: &'a NodePlan, + share: f64, + states: &[crate::budget::State], + want: f64, + now: i64, +) -> Option<(&'a CompiledBudget, Schedule)> { + let mut bound: Option<(&CompiledBudget, Schedule)> = None; + for b in np.node_wide_rates() { + let s = states.iter().find(|s| s.key == b.key); + let sched = admits( + b.max_for(share), + b.window_sub_seconds, + s.map(|s| s.value).unwrap_or(0), + s.and_then(|s| s.expires_at_ms) + .map(|e| e.saturating_sub(now)) + .unwrap_or(0), + want, + now, + ); + // The slowest binds, and "never" beats every number. + let key = |x: &Schedule| x.seconds.unwrap_or(f64::INFINITY); + bound = match bound { + Some(a) if key(&a.1) >= key(&sched) => Some(a), + _ => Some((b, sched)), + }; + } + bound } /// What an item was measured costing, over the counters stream when it is on. @@ -279,6 +305,20 @@ fn assumes( )); } + let conditional: Vec<&str> = np + .budgets + .iter() + .filter(|b| b.when_op.is_some()) + .map(|b| b.id.as_str()) + .collect(); + if !conditional.is_empty() { + parts.push(format!( + "budget {} applies only to selected operations and this queue-level number cannot \ + resolve which operations are ahead", + conditional.join(", ") + )); + } + // §9 closes the caveat list with this one, and it is the one that changes // how the number should be read: a node whose window a breaker has just // spent answers a long `etaSeconds` because a vendor said 429, not because @@ -286,7 +326,7 @@ fn assumes( if let Some(r) = held { parts.push(format!( "a breaker is holding this node until {} ({}s from {}{}), so the window is spent on purpose and this number is that deadline rather than a backlog", - r.at + r.retry_after_seconds * 1000, + r.until_ms(), r.retry_after_seconds, r.at, match &r.by { @@ -387,4 +427,38 @@ mod tests { let s = admits(150, 10, 150, 9_500, 150.0, 0); assert_eq!(s.seconds, Some(9.5)); } + + /// Queue depth has no operation breakdown. Applying a selective limit to + /// every item would produce a confidently late bound for unrelated work. + #[test] + fn a_conditional_budget_does_not_bind_the_whole_queues_eta() { + let doc: gate_core::GraphDoc = serde_json::from_value(serde_json::json!({ + "application": "a", "graph": "g", "version": 1, + "nodes": { "n": { "ingress": true, "egress": "out", + "budgets": [ + { "id": "base", "count": 100, "timeMs": 1000 }, + { "id": "rare", "count": 1, "timeMs": 3600000, + "subWindows": 1, "whenOp": ["photo.delete"] } + ] } }, + "paths": [{ "name": "main", "nodes": ["n"] }] + })) + .expect("document"); + let plan = gate_core::compile(&doc); + let np = plan.node("n").expect("node"); + + let (budget, schedule) = + binding_schedule(np, 1.0, &[], 50.0, 1_000_000).expect("base budget"); + assert_eq!(budget.id, "base"); + assert_eq!(schedule.seconds, Some(0.0)); + } + + #[test] + fn an_extreme_schedule_remains_an_estimate_instead_of_overflowing() { + let s = admits(1, i64::MAX, 1, 10, 3.0, i64::MAX - 5); + assert_eq!(s.resets_at, i64::MAX); + assert!( + s.seconds.is_some_and(|seconds| seconds > i64::MAX as f64), + "the two post-edge windows should be represented without integer overflow: {s:?}" + ); + } } diff --git a/crates/server/src/graph.rs b/crates/server/src/graph.rs index 5cb36ef..59cf96d 100644 --- a/crates/server/src/graph.rs +++ b/crates/server/src/graph.rs @@ -89,46 +89,111 @@ pub async fn declare_locked( let key = doc.key(); let (plan, facts) = compile(app, &doc).await; - let problems = gate_core::validate_with(&doc, &facts); - if !problems.is_empty() { - return Err(Refusal::Invalid(join(&problems))); + // Validate the resolved plan, not a second compilation with library + // defaults. In particular this includes the fleet-wide worker override: + // an unsafe `GATE_STAGE_CONCURRENCY` must be refused before the SDK + // preallocates and spawns that many consumer tasks. + // + // A caller's declare is held to every rule. A document coming back from the + // store is not: it was accepted by some version of Gate and is, in the + // ordinary case, already serving traffic, so a rule added since then must + // not be the thing that takes it down. See `refuses_stored_document`. + let problems = gate_core::validate_plan_with(&doc, &plan, &facts); + let (fatal, kept): (Vec<_>, Vec<_>) = if from_caller { + (problems, Vec::new()) + } else { + problems + .into_iter() + .partition(|p| gate_core::refuses_stored_document(p.rule)) + }; + if !kept.is_empty() { + tracing::warn!( + graph = %key, + rules = %kept.iter().map(|p| p.rule).collect::>().join(", "), + "a stored document breaks a rule this build enforces; it keeps running rather than \ + being taken down, and the next declare of it must fix this: {}", + join(&kept) + ); + } + if !fatal.is_empty() { + return Err(Refusal::Invalid(join(&fatal))); } let old = app.registry.get(&doc.application, &doc.graph); if from_caller { if let Some(old) = &old { - if gate_core::needs_version_bump(&old.doc, &doc) && doc.version <= old.doc.version { - return Err(Refusal::Conflict(format!( - "this change re-founds a counter or strands a queue (a new key starts at zero \ - while the old one counts down its TTL, and work already in an interior queue \ - has no consumer in the new plan): bump version above {}. Drain first — stop \ - pushing, wait for `waitingForBudget` to reach zero on every node, then \ - declare.", - old.doc.version - ))); - } + require_version_bump(&old.doc, &doc)?; } - // The same question of the STORE, because a declare lands on ONE + + // Ask the STORE the same question, because a declare lands on ONE // replica: a graph declared a second ago on another pod is not in this - // registry yet. A store that will not answer is not a reason to refuse — - // the local check still stands. - if let Ok(stored) = crate::store::try_load_all(&app.queen).await { - for other in stored.items.iter().filter(|d| d.key() != key) { - let mine = gate_core::compile(other); - for (node, np) in &mine.nodes { - let Some(q) = &np.ingress_queue else { continue }; - if plan - .nodes - .values() - .any(|n| n.ingress_queue.as_deref() == Some(q.as_str())) - { - return Err(Refusal::Conflict(format!( - "`{q}` is already the ingress of node `{node}` in graph `{}` (declared \ - on another replica). Two consumers of one queue in different groups \ - each get every message, which doubles what leaves.", - other.key() - ))); - } + // registry yet. This is an exact key read rather than the fleet-wide + // prefix scan below, so pagination cannot make an existing graph look + // new. A failed read refuses the mutation: without the predecessor Gate + // cannot prove that replacing it at this version is safe. + match crate::store::load_one(&app.queen, &doc.application, &doc.graph).await { + Ok(Some(stored)) => require_version_bump(&stored, &doc)?, + Ok(None) => {} + Err(e) => { + return Err(Refusal::Gateway(format!( + "`{key}` was not declared: its stored predecessor could not be read ({e}), so \ + Gate cannot safely decide whether this change needs a version bump" + ))) + } + } + // Ask the STORE the same ownership question, because a declare lands on + // ONE replica: a graph declared on another pod need not be in this + // registry yet. This check must fail closed. An error, a clamped page, + // or an unreadable newer document all mean "ownership is unknown", not + // "the source is free". + let stored = crate::store::try_load_all(&app.queen).await.map_err(|e| { + Refusal::Gateway(format!( + "`{key}` was not declared: Gate could not read the stored graph inventory ({e}), \ + so it cannot safely prove exclusive ownership of the source queues" + )) + })?; + // An incomplete inventory only hides an answer for a source Gate does + // NOT name: an owned ingress and an interior queue are derived from + // `{app}.{graph}.{node}`, so no other graph key can mint the same name + // and no unreadable document can be claiming one. A user-declared + // ingress is free-form and another graph really may name it. + // + // Scoping the refusal there matters because `complete` is a fact about + // the whole namespace: `deny_unknown_fields` is deliberate, so ONE + // document written by a newer build makes every declare in every + // application unreadable-and-therefore-refused, which is a rolling + // deploy taking the control plane down for tenants that share nothing + // but a broker. + let user_sources: Vec<&str> = plan + .stages + .iter() + .map(|s| s.source.as_str()) + .filter(|source| { + plan.queue(source) + .is_some_and(|q| q.kind == gate_core::QueueKind::UserIngress) + }) + .collect(); + if !stored.complete && !user_sources.is_empty() { + return Err(Refusal::Gateway(format!( + "`{key}` was not declared: the stored graph inventory is incomplete (a page was \ + clamped or a document could not be read), so Gate cannot prove that {} is not \ + already consumed by another graph. A queue Gate names itself would not need \ + this check.", + user_sources.join(", ") + ))); + } + for other in stored.items.iter().filter(|d| d.key() != key) { + let mine = gate_core::compile(other); + for owner in &mine.stages { + let q = &owner.source; + if plan.stages.iter().any(|candidate| candidate.source == *q) { + return Err(Refusal::Conflict(format!( + "`{q}` is already the source of node `{}` in graph `{}` (declared on \ + another replica). Two consumers of one queue in different groups each \ + get every message, which doubles what leaves.", + owner.node, + other.key() + ))); } } } @@ -256,6 +321,19 @@ fn join(problems: &[Problem]) -> String { .join("; ") } +fn require_version_bump(old: &GraphDoc, new: &GraphDoc) -> Result<(), Refusal> { + if gate_core::needs_version_bump(old, new) && new.version <= old.version { + return Err(Refusal::Conflict(format!( + "this change re-founds a counter or strands a queue (a new key starts at zero while \ + the old one counts down its TTL, and work already in an interior queue has no \ + consumer in the new plan): bump version above {}. Drain first — stop pushing, wait \ + for `waitingForBudget` to reach zero on every node, then declare.", + old.version + ))); + } + Ok(()) +} + /// What the declare answers: the whole compiled plan, so a caller never has to /// reconstruct it and never has to guess a queue name. pub fn resolved(rt: &Arc, warnings: &[Problem]) -> Value { @@ -333,6 +411,7 @@ fn stage_view(s: &gate_core::plan::Stage) -> Value { "node": d.node, "queue": d.queue, "derivesTransactionId": d.derive_id, + "requiresPathStamp": d.requires_stamp, "terminal": d.terminal, })).collect::>(), }) diff --git a/crates/server/src/history.rs b/crates/server/src/history.rs index ac84265..46f334b 100644 --- a/crates/server/src/history.rs +++ b/crates/server/src/history.rs @@ -128,12 +128,21 @@ impl History { /// Add one minute's increments. Two replicas writing the same minute is the /// normal case, not a race: they saw different halves of the traffic, and /// the row is the sum. - pub async fn add(&self, app: &str, target: &str, minute: i64, lanes: &HashMap) { - let Ok(client) = self.pool.get().await else { - return; + pub async fn add( + &self, + app: &str, + target: &str, + minute: i64, + lanes: &HashMap, + ) -> bool { + let Ok(mut client) = self.pool.get().await else { + return false; + }; + let Ok(tx) = client.transaction().await else { + return false; }; for (lane, b) in lanes { - let _ = client + if tx .execute( "INSERT INTO gate.rollups (application, target, lane, minute, admitted, denied, calls, throttled, cost_est, cost_actual) @@ -152,14 +161,26 @@ impl History { &b.cost_estimated, &b.cost_actual, ], ) - .await; + .await + .is_err() + { + return false; + } } + tx.commit().await.is_ok() } - pub async fn rollups(&self, app: &str, target: &str, minutes: i64) -> Vec { - let Ok(client) = self.pool.get().await else { - return vec![]; - }; + pub async fn rollups( + &self, + app: &str, + target: &str, + minutes: i64, + ) -> Result, String> { + let client = self + .pool + .get() + .await + .map_err(|e| format!("history connection: {e}"))?; let since = (crate::now_ms() / 60_000 * 60_000) - minutes * 60_000; let rows = client .query( @@ -170,19 +191,29 @@ impl History { &[&app, &target, &since], ) .await - .unwrap_or_default(); + .map_err(|e| format!("history rollups: {e}"))?; let mut by_minute: Vec<(i64, HashMap, [f64; 6])> = Vec::new(); for r in rows { - let m: i64 = r.get(0); - let lane: String = r.get(1); + let m: i64 = r + .try_get(0) + .map_err(|e| format!("history rollup row: {e}"))?; + let lane: String = r + .try_get(1) + .map_err(|e| format!("history rollup row: {e}"))?; let v = [ - r.get::<_, i64>(2) as f64, - r.get::<_, i64>(3) as f64, - r.get::<_, i64>(4) as f64, - r.get::<_, i64>(5) as f64, - r.get::<_, f64>(6), - r.get::<_, f64>(7), + r.try_get::<_, i64>(2) + .map_err(|e| format!("history rollup row: {e}"))? as f64, + r.try_get::<_, i64>(3) + .map_err(|e| format!("history rollup row: {e}"))? as f64, + r.try_get::<_, i64>(4) + .map_err(|e| format!("history rollup row: {e}"))? as f64, + r.try_get::<_, i64>(5) + .map_err(|e| format!("history rollup row: {e}"))? as f64, + r.try_get::<_, f64>(6) + .map_err(|e| format!("history rollup row: {e}"))?, + r.try_get::<_, f64>(7) + .map_err(|e| format!("history rollup row: {e}"))?, ]; if by_minute.last().map(|(mm, _, _)| *mm) != Some(m) { by_minute.push((m, HashMap::new(), [0.0; 6])); @@ -198,7 +229,7 @@ impl History { ); } - by_minute + Ok(by_minute .into_iter() .map(|(m, lanes, t)| { json!({ @@ -208,15 +239,23 @@ impl History { "lanes": lanes, }) }) - .collect() + .collect()) } /// Admissions per second for one lane, from the table rather than from /// whatever this replica happened to see. - pub async fn rate_per_sec(&self, app: &str, target: &str, lane: &str, now_ms: i64) -> f64 { - let Ok(client) = self.pool.get().await else { - return 0.0; - }; + pub async fn rate_per_sec( + &self, + app: &str, + target: &str, + lane: &str, + now_ms: i64, + ) -> Result { + let client = self + .pool + .get() + .await + .map_err(|e| format!("history connection: {e}"))?; let current = now_ms / 60_000 * 60_000; let rows = client .query( @@ -226,26 +265,26 @@ impl History { &[&app, &target, &lane, &(current - 60_000)], ) .await - .unwrap_or_default(); + .map_err(|e| format!("history rate: {e}"))?; for r in &rows { - let m: i64 = r.get(0); - let a: i64 = r.get(1); + let m: i64 = r.try_get(0).map_err(|e| format!("history rate row: {e}"))?; + let a: i64 = r.try_get(1).map_err(|e| format!("history rate row: {e}"))?; // A complete minute needs no correction. Only fall back to the one // still filling — scaled, with a floor on the divisor — when there // is no complete one yet, which is exactly the first minute after a // declare and exactly when somebody is watching. if m < current { - return a as f64 / 60.0; + return Ok(a as f64 / 60.0); } } - match rows.first() { + Ok(match rows.first() { Some(r) => { - let a: i64 = r.get(1); + let a: i64 = r.try_get(1).map_err(|e| format!("history rate row: {e}"))?; let elapsed = (((now_ms - current) as f64) / 1000.0).max(5.0); a as f64 / elapsed } None => 0.0, - } + }) } /// What one item of this lane charges a budget, measured over the last few @@ -287,47 +326,65 @@ impl History { // counter now, so N ceilings cannot oversubscribe it, and the whole argument // evaporates with the feature. - /// Admissions per minute for every target, over the last `minutes`. + /// Admissions and admitted cost per minute for every target, over the last + /// `minutes`. /// /// One query for the whole deployment rather than one per target: the /// dashboard draws every application at once, and N round trips to draw one /// picture is how a console starts costing more than the thing it watches. - pub async fn flow(&self, minutes: i64, now_ms: i64) -> Vec<(String, String, i64, i64)> { - let Ok(client) = self.pool.get().await else { - return vec![]; - }; + pub async fn flow( + &self, + minutes: i64, + now_ms: i64, + ) -> Result, String> { + let client = self + .pool + .get() + .await + .map_err(|e| format!("history connection: {e}"))?; let since = now_ms / 60_000 * 60_000 - minutes * 60_000; let rows = client .query( - "SELECT application, target, minute, COALESCE(SUM(admitted), 0)::BIGINT + "SELECT application, target, minute, + COALESCE(SUM(admitted), 0)::BIGINT, + COALESCE(NULLIF(SUM(cost_est), 0), SUM(admitted)::DOUBLE PRECISION, 0) FROM gate.rollups WHERE minute >= $1 GROUP BY application, target, minute ORDER BY minute", &[&since], ) .await - .unwrap_or_default(); - rows.iter() - .filter_map(|r| { - Some(( - r.try_get::<_, String>(0).ok()?, - r.try_get::<_, String>(1).ok()?, - r.try_get::<_, i64>(2).ok()?, - r.try_get::<_, i64>(3).ok()?, - )) - }) - .collect() + .map_err(|e| format!("history flow: {e}"))?; + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + out.push(( + r.try_get::<_, String>(0) + .map_err(|e| format!("history flow row: {e}"))?, + r.try_get::<_, String>(1) + .map_err(|e| format!("history flow row: {e}"))?, + r.try_get::<_, i64>(2) + .map_err(|e| format!("history flow row: {e}"))?, + r.try_get::<_, i64>(3) + .map_err(|e| format!("history flow row: {e}"))?, + r.try_get::<_, f64>(4) + .map_err(|e| format!("history flow row: {e}"))?, + )); + } + Ok(out) } - pub async fn add_traces(&self, rows: &[crate::obs::Trace]) { + pub async fn add_traces(&self, rows: &[crate::obs::Trace]) -> bool { if rows.is_empty() { - return; + return true; } - let Ok(client) = self.pool.get().await else { - return; + let Ok(mut client) = self.pool.get().await else { + return false; + }; + let Ok(tx) = client.transaction().await else { + return false; }; for t in rows { - let _ = client + if tx .execute( "INSERT INTO gate.traces (at, application, target, lane, op, outcome, budget_id, calls) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", @@ -336,14 +393,21 @@ impl History { &t.path, &t.op, &t.outcome, &t.budget_id, &0i64, ], ) - .await; + .await + .is_err() + { + return false; + } } + tx.commit().await.is_ok() } - pub async fn traces(&self, outcome: Option<&str>, limit: i64) -> Vec { - let Ok(client) = self.pool.get().await else { - return vec![]; - }; + pub async fn traces(&self, outcome: Option<&str>, limit: i64) -> Result, String> { + let client = self + .pool + .get() + .await + .map_err(|e| format!("history connection: {e}"))?; let rows = match outcome { Some(o) => { client @@ -364,24 +428,32 @@ impl History { .await } } - .unwrap_or_default(); - // `get` panics on a column type it did not expect, and a console page - // is a poor place to discover that somebody upgraded the schema by - // hand. A row that will not read is skipped. - rows.iter() - .filter_map(|r| { - Some(json!({ - "at": r.try_get::<_, i64>(0).ok()?, - "application": r.try_get::<_, String>(1).ok()?, - "target": r.try_get::<_, String>(2).ok()?, - "lane": r.try_get::<_, String>(3).ok()?, - "op": r.try_get::<_, String>(4).ok()?, - "outcome": r.try_get::<_, String>(5).ok()?, - "budget_id": r.try_get::<_, Option>(6).ok()?, - "calls": r.try_get::<_, i64>(7).ok()?, - })) - }) - .collect() + .map_err(|e| format!("history traces: {e}"))?; + // `get` panics on a column type it did not expect. A schema mismatch + // must be visible as a failed read rather than quietly shortening the + // result set and making an incident disappear from the console. + let mut out = Vec::with_capacity(rows.len()); + for r in rows { + out.push(json!({ + "at": r.try_get::<_, i64>(0) + .map_err(|e| format!("history trace row: {e}"))?, + "application": r.try_get::<_, String>(1) + .map_err(|e| format!("history trace row: {e}"))?, + "target": r.try_get::<_, String>(2) + .map_err(|e| format!("history trace row: {e}"))?, + "lane": r.try_get::<_, String>(3) + .map_err(|e| format!("history trace row: {e}"))?, + "op": r.try_get::<_, String>(4) + .map_err(|e| format!("history trace row: {e}"))?, + "outcome": r.try_get::<_, String>(5) + .map_err(|e| format!("history trace row: {e}"))?, + "budget_id": r.try_get::<_, Option>(6) + .map_err(|e| format!("history trace row: {e}"))?, + "calls": r.try_get::<_, i64>(7) + .map_err(|e| format!("history trace row: {e}"))?, + })); + } + Ok(out) } /// Retention, on its own slow clock. `O(space)` work does not belong on the diff --git a/crates/server/src/knobs.rs b/crates/server/src/knobs.rs index 106fdf8..aa405a1 100644 --- a/crates/server/src/knobs.rs +++ b/crates/server/src/knobs.rs @@ -122,6 +122,30 @@ pub struct Knobs { /// (`004_log_pop.sql`), so an explicit failed ack is reserved for real /// poison and a retry budget means what it says. pub retry_limit: i32, + /// The largest body a PUSH route will buffer, in bytes. + /// + /// axum's own default is 2 MiB and nothing here ever overrode it, so that + /// number was the real ceiling on everything a caller can hand this service + /// — silently, because the refusal it produces says + /// `Failed to buffer the request body: length limit exceeded` and names + /// neither the limit nor the fact that it is ours. + /// + /// It was measured from prod on 2026-09-04, from both sides of the wall, by + /// a caller that had spent a week failing against it: pushes of 11,408 / + /// 10,387 / 8,976 records went through, and pushes of 12,000 and 16,096 did + /// not. Divide, and 2 MiB is exactly where those cross — the payloads run + /// about 130 to 175 bytes a record depending on the vendor. + /// + /// 8 MiB, and the ceiling on the ceiling is memory rather than taste: the + /// service runs with a 512 MiB limit, and a body limit is a per-request + /// buffer. Four times the old value keeps a large caller comfortably inside + /// it while a burst of ten concurrent pushes still costs under a sixth of + /// the pod. + /// + /// PUSH ROUTES ONLY. Declaring a graph or reading the console has no reason + /// to accept megabytes, and axum's default is the right answer everywhere + /// the body is a document rather than a batch. + pub max_push_body: usize, } impl Default for Knobs { @@ -140,10 +164,28 @@ impl Default for Knobs { max_prefix_retries: 2, interior_seed_skew: crate::relay::INTERIOR_SEED_SKEW, retry_limit: 3, + max_push_body: 8 * 1024 * 1024, } } } +/// axum's own `DefaultBodyLimit`, which applied to every route here until +/// 2026-09-04 because nothing set one. Named so the floor below says why it is +/// where it is. +pub const AXUM_DEFAULT_BODY_LIMIT: usize = 2 * 1024 * 1024; + +/// The largest a push body may be configured to be. +/// +/// The limit is a per-request memory reservation, not a per-request cost: the +/// buffered bytes, the `serde_json::Value` they parse into, the copy the +/// envelope is built on and the body sent to the broker are all live at once, +/// and there is no concurrency limiter in front of any of it. A typo in a +/// deployment manifest should not be able to ask one pod to hold gigabytes. +/// +/// 64 MiB is eight times the default and far past any real push; a deployment +/// that wants more than this wants a different shape, not a bigger number. +pub const MAX_PUSH_BODY_CEILING: usize = 64 * 1024 * 1024; + fn env_u32(name: &str) -> Option { std::env::var(name).ok().and_then(|v| v.parse().ok()) } @@ -182,6 +224,12 @@ pub fn knobs() -> &'static Knobs { retry_limit: env_u32("GATE_RETRY_LIMIT") .map(|n| n as i32) .unwrap_or(d.retry_limit), + // Floored at axum's own default rather than at zero: a typo in the + // environment must not be able to make this service refuse bodies it + // accepted before anybody set the variable. + max_push_body: env_u32("GATE_MAX_PUSH_BODY_BYTES") + .map(|n| (n as usize).clamp(AXUM_DEFAULT_BODY_LIMIT, MAX_PUSH_BODY_CEILING)) + .unwrap_or(d.max_push_body), } }) } diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index bb42fa4..7c89354 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -42,7 +42,7 @@ pub fn now_ms() -> i64 { .unwrap_or(0) } -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use queen_mq::{Config, Queen}; @@ -112,6 +112,7 @@ pub async fn run() -> Result<(), Box> { queen, registry: Default::default(), depths: Arc::new(depth::Depths::default()), + backlogs: Default::default(), traces: Arc::new(obs::Traces::default()), history: history.clone(), queen_url: queen_url.clone(), @@ -208,68 +209,131 @@ pub fn spawn_reconcile( /// It reads the stages' own `AtomicU64`s and writes the DELTA since the last /// pass, so two replicas writing the same minute is the normal case rather than /// a race: they saw different halves of the traffic and the row is the sum. +type CounterSnapshot = (u64, u64, u64); +type CounterCheckpoint = (String, CounterSnapshot); +const MINUTE_MS: i64 = gate_core::COUNTERS_WINDOW_SECONDS as i64 * 1_000; + pub fn spawn_counters(app: api::Shared) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { - let mut last: HashMap = HashMap::new(); + let mut last: HashMap = HashMap::new(); loop { - tokio::time::sleep(std::time::Duration::from_secs(60)).await; - let now = now_ms(); - let minute = now / 60_000 * 60_000; + // Anchor samples to wall-clock minute boundaries. Sleeping a fixed + // minute from process start makes every bucket span, for example, + // 12:00:37..12:01:37 while labelling it 12:01:00. + tokio::time::sleep(until_next_minute(now_ms())).await; + // The delta ends at this boundary, so it belongs to the minute that + // just completed, not to the empty minute that has just begun. + let minute = completed_minute(now_ms()); if let Some(h) = app.history.as_ref() { + let mut active = HashSet::new(); for g in app.registry.all() { if g.plan.counters_window_seconds.is_none() { continue; } let mut per_target: HashMap> = HashMap::new(); + let mut checkpoints: HashMap> = HashMap::new(); for s in &g.stages { let key = format!("{}/{}", g.key(), s.key()); + active.insert(key.clone()); + let target = format!("{}.{}", g.doc.graph, s.stage.node); let c = &s.counters; let o = std::sync::atomic::Ordering::Relaxed; let now3 = ( c.admitted.load(o), - c.deferred.load(o) + c.released.load(o), + c.deferred.load(o).saturating_add(c.released.load(o)), c.cost.load(o), ); - let was = last.insert(key, now3).unwrap_or((0, 0, 0)); - let d = ( - now3.0.saturating_sub(was.0), - now3.1.saturating_sub(was.1), - now3.2.saturating_sub(was.2), - ); + let d = counter_delta(now3, last.get(&key).copied()); + checkpoints + .entry(target.clone()) + .or_default() + .push((key, now3)); if d == (0, 0, 0) { continue; } - per_target - .entry(format!("{}.{}", g.doc.graph, s.stage.node)) - .or_default() - .insert( - s.stage.path.clone(), - history::Bucket { - admitted: d.0, - denied: d.1, - cost_estimated: d.2 as f64, - ..Default::default() - }, - ); + per_target.entry(target).or_default().insert( + s.stage.path.clone(), + history::Bucket { + admitted: d.0, + denied: d.1, + cost_estimated: d.2 as f64, + ..Default::default() + }, + ); } - for (target, paths) in per_target { - h.add(&g.doc.application, &target, minute, &paths).await; + for (target, samples) in checkpoints { + let written = match per_target.remove(&target) { + Some(paths) => h.add(&g.doc.application, &target, minute, &paths).await, + None => true, + }; + if written { + for (key, value) in samples { + last.insert(key, value); + } + } else { + tracing::warn!( + application = %g.doc.application, + %target, + "could not persist counter rollup; retaining the previous checkpoint" + ); + } } } + // A delete, or a redeclare that removes a path, must also + // remove its lifetime-counter baseline. Otherwise this map + // grows for the life of the process and a later stage reusing + // the same identity inherits a checkpoint from a runtime that + // no longer exists. Failed writes for ACTIVE stages remain in + // the set and deliberately retain their previous checkpoint. + retain_active_checkpoints(&mut last, &active); // The refusal ring, on the same cadence. Bounded and // drop-oldest, so a flush that misses a pass loses the oldest // denials and never blocks the hot path. let traces = app.traces.drain(); - if !traces.is_empty() { - h.add_traces(&traces).await; + if !traces.is_empty() && !h.add_traces(&traces).await { + tracing::warn!( + count = traces.len(), + "could not persist traces; returning them to the ring" + ); + app.traces.restore(traces); } } } }) } +fn retain_active_checkpoints( + checkpoints: &mut HashMap, + active: &HashSet, +) { + checkpoints.retain(|key, _| active.contains(key)); +} + +/// Delta of a lifetime counter tuple. A lower value means the stage runtime was +/// replaced and its atomics restarted at zero, so the new value is itself the +/// entire increment since that reset. +fn counter_delta(now: CounterSnapshot, previous: Option) -> CounterSnapshot { + let Some(was) = previous else { + return now; + }; + ( + now.0.checked_sub(was.0).unwrap_or(now.0), + now.1.checked_sub(was.1).unwrap_or(now.1), + now.2.checked_sub(was.2).unwrap_or(now.2), + ) +} + +fn until_next_minute(now_ms: i64) -> std::time::Duration { + let elapsed = now_ms.rem_euclid(MINUTE_MS); + std::time::Duration::from_millis((MINUTE_MS - elapsed) as u64) +} + +fn completed_minute(now_ms: i64) -> i64 { + now_ms.div_euclid(MINUTE_MS) * MINUTE_MS - MINUTE_MS +} + /// Bring back everything that was declared, at boot. pub async fn restore(app: &api::Shared) { let _guard = app.declare_lock.lock().await; @@ -332,7 +396,15 @@ pub async fn reconcile(app: &api::Shared) { // matters: a graph whose provisioning failed is registered-and- // stopped, and comparing documents alone would leave it down for // ever while its ingress queue kept filling. - Some(doc) if doc == rt.doc && rt.is_running() => {} + Some(doc) if doc == rt.doc && rt.is_running() => { + // Seeing this exact document in the authoritative store is + // proof that it is durable, even if this replica never saw the + // answer to its own write. Without repairing the marker here, + // a later delete on another replica is mistaken for a failed + // initial save and this replica resurrects the graph. + rt.persisted + .store(true, std::sync::atomic::Ordering::Relaxed); + } Some(doc) => { tracing::info!(graph = %doc.key(), "reconcile: re-declaring a graph that is changed or not fully up"); if let Err(e) = graph::declare_from_store(app, doc).await { @@ -364,3 +436,52 @@ pub async fn reconcile(app: &api::Shared) { } } } + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + + use super::{completed_minute, counter_delta, retain_active_checkpoints, until_next_minute}; + + #[test] + fn a_restarted_counter_counts_from_its_new_zero() { + assert_eq!(counter_delta((7, 3, 11), Some((100, 80, 900))), (7, 3, 11)); + } + + #[test] + fn a_running_counter_reports_only_its_increment() { + assert_eq!( + counter_delta((107, 83, 911), Some((100, 80, 900))), + (7, 3, 11) + ); + } + + #[test] + fn counter_flush_waits_for_the_next_wall_clock_boundary() { + assert_eq!(until_next_minute(120_000).as_millis(), 60_000); + assert_eq!(until_next_minute(120_001).as_millis(), 59_999); + assert_eq!(until_next_minute(179_999).as_millis(), 1); + } + + #[test] + fn counter_flush_labels_the_minute_that_just_completed() { + assert_eq!(completed_minute(120_000), 60_000); + assert_eq!(completed_minute(120_001), 60_000); + assert_eq!(completed_minute(179_999), 60_000); + assert_eq!(completed_minute(180_000), 120_000); + } + + #[test] + fn deleted_stages_do_not_leave_lifetime_checkpoints_behind() { + let mut checkpoints = HashMap::from([ + ("app/graph/path/live".to_string(), (10, 20, 30)), + ("app/graph/path/deleted".to_string(), (40, 50, 60)), + ]); + let active = HashSet::from(["app/graph/path/live".to_string()]); + + retain_active_checkpoints(&mut checkpoints, &active); + + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints["app/graph/path/live"], (10, 20, 30)); + } +} diff --git a/crates/server/src/obs.rs b/crates/server/src/obs.rs index 0e3ccb6..9303b30 100644 --- a/crates/server/src/obs.rs +++ b/crates/server/src/obs.rs @@ -13,8 +13,9 @@ //! Nothing was broken; that is what the observability of the old design cost //! while idle, and idle is most of the time. -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; use parking_lot::RwLock; use serde_json::{json, Value}; @@ -112,6 +113,69 @@ impl StageCounters { } } +/// Consecutive live backlog samples for the overview's `saturating` state. +/// A growth observation is held across a few console polls so the state does +/// not flicker between the depth cache's refreshes. +const SATURATING_HOLD: Duration = Duration::from_secs(15); + +#[derive(Debug)] +struct BacklogSample { + depth: u64, + /// Whether the PREVIOUS sample was already higher than the one before it. + /// One reading above the last is queue jitter — a backlog that never sits + /// at zero oscillates between polls — and latching on it holds a healthy + /// graph in "needs attention" for ever. Two in a row is the shortest + /// sequence that can distinguish a rise from a wobble. + rising: bool, + growing_until: Option, +} + +#[derive(Default)] +pub struct BacklogTrends { + samples: RwLock>, +} + +impl BacklogTrends { + pub fn sample(&self, graph: &str, depth: u64) -> bool { + self.sample_at(graph, depth, Instant::now()) + } + + /// Forget the graphs that are gone. Nothing else removes an entry, and the + /// key is a graph name, so without this a deployment that declares and + /// deletes graphs grows this map for the life of the process. + pub fn retain(&self, live: &std::collections::HashSet) { + let mut samples = self.samples.write(); + if samples.len() > live.len() { + samples.retain(|graph, _| live.contains(graph)); + } + } + + fn sample_at(&self, graph: &str, depth: u64, now: Instant) -> bool { + let mut samples = self.samples.write(); + let Some(previous) = samples.get_mut(graph) else { + samples.insert( + graph.to_string(), + BacklogSample { + depth, + rising: false, + growing_until: None, + }, + ); + return false; + }; + + let up = depth > previous.depth; + if up && previous.rising { + previous.growing_until = Some(now + SATURATING_HOLD); + } else if depth < previous.depth || depth == 0 { + previous.growing_until = None; + } + previous.rising = up; + previous.depth = depth; + previous.growing_until.is_some_and(|until| until > now) + } +} + /// One refusal, kept. /// /// **Denials only.** An admission is counted and never traced: it is the common @@ -143,6 +207,9 @@ impl Trace { "path": self.path, "op": self.op, "outcome": self.outcome, + // Durable traces have always used the schema/API spelling. Keep the + // former live-only camelCase alias for one compatibility window. + "budget_id": self.budget_id, "budgetId": self.budget_id, }) } @@ -183,6 +250,18 @@ impl Traces { .collect() } + /// Put a failed durable flush back before anything recorded while the write + /// was in flight. The ring remains bounded and drops its oldest entries. + pub fn restore(&self, rows: Vec) { + let mut ring = self.ring.write(); + for row in rows.into_iter().rev() { + ring.push_front(row); + } + while ring.len() > TRACE_RING { + ring.pop_front(); + } + } + pub fn len(&self) -> usize { self.ring.read().len() } @@ -191,3 +270,90 @@ impl Traces { self.len() == 0 } } + +#[cfg(test)] +mod tests { + use super::*; + + fn trace(at: i64) -> Trace { + Trace { + at, + application: "a".into(), + graph: "g".into(), + node: "n".into(), + path: "p".into(), + op: String::new(), + outcome: "denied", + budget_id: Some("b".into()), + } + } + + #[test] + fn a_failed_trace_flush_returns_before_newer_rows() { + let traces = Traces::default(); + traces.push(trace(1)); + traces.push(trace(2)); + let failed = traces.drain(); + traces.push(trace(3)); + + traces.restore(failed); + let recent: Vec = traces.recent(None, 3).iter().map(|t| t.at).collect(); + assert_eq!(recent, vec![3, 2, 1]); + } + + #[test] + fn backlog_growth_is_held_but_drain_clears_it_immediately() { + let trends = BacklogTrends::default(); + let now = Instant::now(); + + assert!(!trends.sample_at("app/g", 3, now)); + assert!(!trends.sample_at("app/g", 5, now + Duration::from_secs(1))); + assert!(trends.sample_at("app/g", 7, now + Duration::from_secs(2))); + assert!(trends.sample_at("app/g", 7, now + Duration::from_secs(6))); + assert!(!trends.sample_at("app/g", 4, now + Duration::from_secs(7))); + assert!(!trends.sample_at("app/g", 0, now + Duration::from_secs(8))); + } + + #[test] + fn a_growth_latch_expires_without_another_increase() { + let trends = BacklogTrends::default(); + let now = Instant::now(); + + assert!(!trends.sample_at("app/g", 1, now)); + assert!(!trends.sample_at("app/g", 2, now + Duration::from_secs(1))); + assert!(trends.sample_at("app/g", 3, now + Duration::from_secs(2))); + assert!(!trends.sample_at("app/g", 3, now + Duration::from_secs(18))); + } + + /// A busy graph's backlog is never zero and never still. One reading above + /// the last is the normal shape of a queue that is being drained as fast as + /// it fills, and it must not pin the graph in "needs attention". + #[test] + fn an_oscillating_backlog_is_not_saturating() { + let trends = BacklogTrends::default(); + let now = Instant::now(); + + let mut at = now; + for (i, depth) in [100, 120, 100, 120, 100, 120, 100].into_iter().enumerate() { + at = now + Duration::from_secs(i as u64); + assert!( + !trends.sample_at("app/g", depth, at), + "sample {i} at depth {depth} latched on jitter" + ); + } + // A real rise still shows, on the second consecutive increase. + assert!(!trends.sample_at("app/g", 130, at + Duration::from_secs(1))); + assert!(trends.sample_at("app/g", 160, at + Duration::from_secs(2))); + } + + #[test] + fn a_graph_that_is_gone_is_forgotten() { + let trends = BacklogTrends::default(); + trends.sample("app/kept", 1); + trends.sample("app/gone", 1); + + trends.retain(&["app/kept".to_string()].into_iter().collect()); + assert_eq!(trends.samples.read().len(), 1); + assert!(trends.samples.read().contains_key("app/kept")); + } +} diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index 7fd8bc7..563dcc4 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -42,7 +42,7 @@ pub struct GraphRuntime { /// is stop-then-start, so the state exists for as long as a swap takes and /// outlives it whenever a restore fails — this is what makes it visible /// instead of implied. - pub stopped: AtomicBool, + pub stopped: Arc, /// One token per graph, cloned into every stage. pub cancel: queen_mq::Cancel, } @@ -143,17 +143,20 @@ impl Registry { } } - /// Ingress queues already claimed, excluding one graph — what the - /// `ingress-owner` rule is asked against on a redeclare. + /// Stage source queues already claimed, excluding one graph — what the + /// `ingress-owner` rule is asked against on a redeclare. This includes + /// Gate-owned interior queues: allowing another graph to call one a + /// user-owned ingress gives the same physical stream a second consumer. pub fn ingress_owners(&self, except: &str) -> Vec<(String, String, String)> { let mut out = Vec::new(); for g in self.all() { if g.key() == except { continue; } - for (name, np) in &g.plan.nodes { - if let Some(q) = &np.ingress_queue { - out.push((q.clone(), g.key(), name.clone())); + let mut seen = std::collections::HashSet::new(); + for stage in &g.plan.stages { + if seen.insert(stage.source.clone()) { + out.push((stage.source.clone(), g.key(), stage.node.clone())); } } } diff --git a/crates/server/src/relay.rs b/crates/server/src/relay.rs index 845e732..0bfa73a 100644 --- a/crates/server/src/relay.rs +++ b/crates/server/src/relay.rs @@ -40,7 +40,7 @@ //! rotation cursor and `MAX_IN_FLIGHT` all existed to do, badly, what the broker //! does here for free. -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -50,7 +50,7 @@ use queen_mq::{Cancel, Message, Queen, SubscriptionMode, TxnPushItem}; use serde_json::{json, Value}; use gate_core::plan::{NodePlan, Stage}; -use gate_core::{cost_of, op_matches, op_of, scope_value, GATE_META}; +use gate_core::{cost_of, missing_scope, op_matches, op_of, scope_value, GATE_META}; use crate::budget::{Budgets, Charge, Ledger}; use crate::knobs::knobs; @@ -228,6 +228,7 @@ pub fn spawn( budgets: Budgets, st: Arc, traces: Arc, + graph_stopped: Arc, ) -> tokio::task::JoinHandle<()> { let k = knobs(); let q = queen.clone(); @@ -276,10 +277,16 @@ pub fn spawn( } }) .await; + // A consumer normally returns only because its graph was deliberately + // cancelled. Any other return leaves this stage's source without a + // reader, so fail the whole runtime closed and stop its siblings. The + // reconcile loop can then see `is_running() == false` and replace it. + let unexpected = stop_graph_after_stage_exit(&graph_stopped, &st.cancel); match res { Ok(summary) => tracing::info!( stage = %st.key(), queue = %st.stage.source, processed = summary.processed, reason = ?summary.stopped_by, + unexpected, "stage stopped" ), // A stage that exits is a stopped graph, which is the failure v1's @@ -296,6 +303,17 @@ pub fn spawn( }) } +/// Mark a runtime unhealthy when a stage returns without a graph cancellation. +/// Returns whether this exit initiated the stop, for the terminal log line. +fn stop_graph_after_stage_exit(stopped: &AtomicBool, cancel: &queen_mq::Cancel) -> bool { + if cancel.is_cancelled() { + return false; + } + stopped.store(true, Ordering::Relaxed); + cancel.cancel(); + true +} + struct Ctx { queen: Queen, budgets: Budgets, @@ -361,21 +379,7 @@ async fn handle(ctx: &Ctx, msgs: Vec) { return; } - let kinds: Vec = msgs - .iter() - .map(|m| { - // §6.7. Three groups read `ip.in` in the flagship graph and each sees - // every message; only the one whose `_gate.path` matches forwards it. - // The others must SETTLE it or their cursor never advances. - if st.stage.check_foreign && !owns(&st.stage, &m.data) { - return Kind::Foreign; - } - match cost_of(&st.node.cost, &m.data) { - Ok(_) => Kind::Work, - Err(e) => Kind::Poison(format!("gate: node `{}`: {e}", st.node.name)), - } - }) - .collect(); + let kinds: Vec = msgs.iter().map(|m| classify(st, m)).collect(); // A poison message at the HEAD is nacked on its own, because a nack and an // ack in one transaction contradict each other: the nack releases the lease @@ -413,6 +417,44 @@ async fn handle(ctx: &Ctx, msgs: Vec) { admit(ctx, &msgs[..cut], &kinds[..cut]).await; } +fn classify(st: &StageRuntime, message: &Message) -> Kind { + // §6.7. Three groups read `ip.in` in the flagship graph and each sees every + // message; only the one whose `_gate.path` matches forwards it. The others + // must SETTLE it or their cursor never advances, even when its payload would + // be invalid for this other path. + if st.stage.check_foreign && !owns(&st.stage, &message.data) { + return Kind::Foreign; + } + // A shared interior queue has one consumer group per path. Those groups + // can tell their frames apart only from `_gate.path`, and a JSON scalar or + // array cannot carry that stamp. Forwarding one anyway makes the compiler's + // arbitrary unstamped owner charge and route every copy as its own path. + // Refuse it at the last unambiguous stage instead. Linear and terminal + // routes remain shape-preserving because neither needs path provenance. + if !message.data.is_object() + && st + .stage + .destinations + .iter() + .any(|destination| destination.requires_stamp) + { + return Kind::Poison(format!( + "gate: node `{}`: payload must be a JSON object before a shared interior queue", + st.node.name + )); + } + if let Err(error) = cost_of(&st.node.cost, &message.data) { + return Kind::Poison(format!("gate: node `{}`: {error}", st.node.name)); + } + if let Some((budget, path)) = missing_scope(&st.node.budgets, &message.data) { + return Kind::Poison(format!( + "gate: node `{}`: budget `{budget}` counts per `{path}` and this item carries none", + st.node.name + )); + } + Kind::Work +} + /// §6.1 – §6.5. `window` is in offset order and all from one source partition. async fn admit(ctx: &Ctx, window: &[Message], kinds: &[Kind]) { let st = &ctx.st; @@ -721,18 +763,35 @@ impl Grouped { /// touches is dropped rather than charged zero. fn charges(&self, n: usize) -> Vec { let mut deltas = vec![0i64; self.keys.len()]; + let mut overflowed = vec![false; self.keys.len()]; for contributions in self.per_msg.iter().take(n) { for (idx, cost) in contributions { - deltas[*idx] += cost; + match deltas[*idx].checked_add(*cost) { + Some(total) => deltas[*idx] = total, + None => { + // The wire cannot express this batch's total. Ask for a + // deliberately impossible increment so the ordinary + // refusal path computes the largest representable + // prefix instead of wrapping the delta and admitting it. + deltas[*idx] = i64::MAX; + overflowed[*idx] = true; + } + } } } self.keys .iter() - .zip(deltas.iter()) - .filter(|(_, d)| **d > 0) - .map(|(k, d)| Charge { + .zip(deltas.iter().zip(overflowed.iter())) + .filter(|(_, (d, _))| **d > 0) + .map(|(k, (d, overflowed))| Charge { key: k.key.clone(), - max: k.max, + // With the largest legal ceiling, delta == max would otherwise + // apply even though the true (unrepresentable) sum is larger. + max: if *overflowed { + k.max.min(i64::MAX - 1) + } else { + k.max + }, ttl: k.ttl, delta: *d, budget_id: k.budget_id.clone(), @@ -761,18 +820,19 @@ impl Grouped { .map(|s| s.value) .unwrap_or(0) .saturating_sub(mine); - remaining[i] = (key.max - current).max(0); + remaining[i] = key.max.saturating_sub(current).max(0); } let mut used = vec![0i64; self.keys.len()]; for (n, contributions) in self.per_msg.iter().enumerate() { for (idx, cost) in contributions { - if used[*idx] + cost > remaining[*idx] { + let Some(total) = used[*idx].checked_add(*cost) else { + return n; + }; + if total > remaining[*idx] { return n; } - } - for (idx, cost) in contributions { - used[*idx] += cost; + used[*idx] = total; } } self.per_msg.len() @@ -796,12 +856,9 @@ fn group(st: &StageRuntime, msgs: &[Message]) -> Grouped { let key = match &b.scope_by { Some(path) => match scope_value(&m.data, path) { Some(v) => b.key_for(Some(&v)), - // A counter keyed on an absent value measures the wrong - // thing. The HTTP front door refuses this with a 422; an - // item that arrived on a user-owned ingress queue without it - // is charged against the node's other budgets and skips this - // one, because dropping the item would be a limiter losing - // work it was asked to pace. + // `classify` makes an applicable missing scope poison before + // grouping. Keep this defensive branch for recomputed + // refunds, which must never invent a scope key. None => continue, }, None => b.key.clone(), @@ -818,7 +875,13 @@ fn group(st: &StageRuntime, msgs: &[Message]) -> Grouped { keys.len() - 1 } }; - here.push((idx, cost)); + // A shared key is one counter even when more than one budget on + // this node names it. The validator permits identical declarations + // deliberately; charging the same key once per declaration would + // multiply this message's cost and enforce a smaller limit. + if !here.iter().any(|(seen, _)| *seen == idx) { + here.push((idx, cost)); + } } per_msg.push(here); } @@ -1079,7 +1142,7 @@ async fn stage_and_commit( continue; } forwarded += 1; - cost += cost_of(&st.node.cost, &m.data).unwrap_or(1); + cost = cost.saturating_add(cost_of(&st.node.cost, &m.data).unwrap_or(1)); for dest in &st.stage.destinations { match tx.push_item(push_for(st, m, dest)) { Ok(next) => tx = next, @@ -1100,9 +1163,13 @@ async fn stage_and_commit( st.counters.admitted.fetch_add(forwarded, Ordering::Relaxed); st.counters.foreign.fetch_add(foreign, Ordering::Relaxed); st.counters.commits.fetch_add(1, Ordering::Relaxed); - st.counters + let delta = cost.max(0) as u64; + let _ = st + .counters .cost - .fetch_add(cost.max(0) as u64, Ordering::Relaxed); + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + Some(value.saturating_add(delta)) + }); // The cursor moved, so whatever was being counted at the old head is // over. The head comparison in `note_failed_settle` would notice on // its own; this keeps the count honest without waiting for a second @@ -1138,8 +1205,15 @@ fn push_for(st: &StageRuntime, m: &Message, dest: &gate_core::Destination) -> Tx async fn settle_head(ctx: &Ctx, m: &Message, kind: &Kind) -> bool { let st = &ctx.st; if !matches!(kind, Kind::Work) { - let _ = ctx.queen.transaction().ack(m).commit().await; - st.counters.foreign.fetch_add(1, Ordering::Relaxed); + match ctx.queen.transaction().ack(m).commit().await { + Ok(_) => { + st.counters.foreign.fetch_add(1, Ordering::Relaxed); + } + Err(e) => tracing::warn!( + stage = %st.key(), error = %e, + "could not settle a foreign item at the head; its lease will lapse" + ), + } return false; } @@ -1153,13 +1227,21 @@ async fn settle_head(ctx: &Ctx, m: &Message, kind: &Kind) -> bool { } let Some(tx) = tx else { // Nack with the reason so it reaches the DLQ, never dropped. - let _ = ctx + match ctx .queen .transaction() .nack(m, "gate: this item cannot be staged for its destination") .commit() - .await; - st.counters.deadlettered.fetch_add(1, Ordering::Relaxed); + .await + { + Ok(_) => { + st.counters.deadlettered.fetch_add(1, Ordering::Relaxed); + } + Err(e) => tracing::warn!( + stage = %st.key(), error = %e, + "could not dead-letter an item that cannot be staged; its lease will lapse" + ), + } return false; }; match tx.commit().await { @@ -1167,6 +1249,17 @@ async fn settle_head(ctx: &Ctx, m: &Message, kind: &Kind) -> bool { st.counters.forwarded.fetch_add(1, Ordering::Relaxed); st.counters.admitted.fetch_add(1, Ordering::Relaxed); st.counters.commits.fetch_add(1, Ordering::Relaxed); + // The same weight the batch path records, and for the same reason: + // the budget was charged for this item, so a roll-up that counts the + // admission and not the cost measures a node as idler than it is. + // Utilisation is read from this number. + let cost = cost_of(&st.node.cost, &m.data).unwrap_or(1).max(0) as u64; + let _ = st + .counters + .cost + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + Some(value.saturating_add(cost)) + }); true } // Already downstream: settle it and move on. It does NOT count as @@ -1302,6 +1395,26 @@ fn jitter_ms(wait_ms: i64) -> i64 { mod tests { use super::*; + #[test] + fn an_unexpected_stage_exit_stops_the_graph_and_its_siblings() { + let stopped = AtomicBool::new(false); + let cancel = Cancel::new(); + + assert!(stop_graph_after_stage_exit(&stopped, &cancel)); + assert!(stopped.load(Ordering::Relaxed)); + assert!(cancel.is_cancelled()); + } + + #[test] + fn a_planned_stage_exit_does_not_reclassify_the_stop() { + let stopped = AtomicBool::new(false); + let cancel = Cancel::new(); + cancel.cancel(); + + assert!(!stop_graph_after_stage_exit(&stopped, &cancel)); + assert!(!stopped.load(Ordering::Relaxed)); + } + fn budget(id: &str, count_sub: i64) -> gate_core::CompiledBudget { gate_core::CompiledBudget { id: id.into(), @@ -1393,6 +1506,61 @@ mod tests { assert_eq!(charges[0].max, 100); } + /// Two identical declarations may intentionally share one counter. They + /// remain one spend per message, not one spend per declaration. + #[test] + fn duplicate_shared_budgets_charge_their_counter_once() { + let mut first = budget("first", 100); + first.shared_key = Some("vendor".into()); + let mut second = budget("second", 100); + second.key = first.key.clone(); + second.shared_key = first.shared_key.clone(); + let st = runtime(vec![first, second], 1.0); + let msgs: Vec = (0..3) + .map(|i| msg(&format!("t{i}"), json!({ "w": 4 }))) + .collect(); + + let charges = group(&st, &msgs).charges(msgs.len()); + assert_eq!(charges.len(), 1, "one shared key must produce one incr"); + assert_eq!(charges[0].delta, 12, "the declarations doubled the cost"); + } + + /// `kv.incr` carries an i64 delta. A valid variable-cost declaration can + /// still put two individually legal values in one batch whose sum is above + /// that wire ceiling; it must take the refusal/prefix path, never wrap to a + /// small or negative charge. + #[test] + fn an_unrepresentable_batch_cost_fails_closed_to_a_prefix() { + let mut st = runtime(vec![budget("b", i64::MAX)], 1.0); + st.node.cost = gate_core::Cost::Path(gate_core::CostPath { + path: "payload.w".into(), + default: 1, + max: Some(i64::MAX), + }); + let item_cost = i64::MAX / 2 + 1; + let msgs = vec![ + msg("t0", json!({ "w": item_cost })), + msg("t1", json!({ "w": item_cost })), + ]; + let grouped = group(&st, &msgs); + let charges = grouped.charges(2); + + assert_eq!(charges[0].delta, i64::MAX); + assert_eq!(charges[0].max, i64::MAX - 1, "the overflow must refuse"); + + let attempt = crate::budget::Attempt { + applied: vec![false], + post: vec![None], + states: vec![crate::budget::State { + key: charges[0].key.clone(), + value: 0, + expires_at_ms: None, + }], + }; + assert_eq!(grouped.prefix(&charges, &attempt), 1); + assert_eq!(grouped.charges(1)[0].delta, item_cost); + } + /// A path's share IS the ceiling it carries: `round(count_sub * share)`. #[test] fn the_share_is_the_max_on_the_incr() { @@ -1512,6 +1680,26 @@ mod tests { assert_eq!(l1.delta, 2); } + #[test] + fn an_applicable_missing_scope_is_poison_on_direct_ingress() { + let mut scoped = budget("per-listing", 100); + scoped.scope_by = Some("payload.listingId".into()); + scoped.when_op = Some(vec!["photo.delete".into()]); + let st = runtime(vec![budget("all", 100), scoped], 1.0); + + let missing = msg("t0", json!({ "w": 1, "op": "photo.delete" })); + match classify(&st, &missing) { + Kind::Poison(reason) => { + assert!(reason.contains("per-listing"), "{reason}"); + assert!(reason.contains("payload.listingId"), "{reason}"); + } + _ => panic!("a missing applicable scope was allowed through"), + } + + let unrelated = msg("t1", json!({ "w": 1, "op": "photo.upload" })); + assert!(matches!(classify(&st, &unrelated), Kind::Work)); + } + fn stage_named(path: &str, owns_unstamped: bool) -> Stage { let mut s = runtime(vec![budget("b", 10)], 1.0).stage; s.path = path.into(); @@ -1551,6 +1739,34 @@ mod tests { assert!(!owns(&other, &json!({}))); } + #[test] + fn an_unstampable_payload_never_enters_a_shared_interior_queue() { + let mut st = runtime(vec![budget("b", 10)], 1.0); + st.stage.destinations.push(gate_core::Destination { + node: "next".into(), + queue: "shared.in".into(), + label: "app/g/p/next".into(), + derive_id: true, + requires_stamp: true, + terminal: false, + }); + + match classify(&st, &msg("scalar", json!("cannot carry _gate"))) { + Kind::Poison(reason) => assert!(reason.contains("JSON object"), "{reason}"), + _ => panic!("an unstampable payload was allowed into a shared queue"), + } + assert!(matches!( + classify(&st, &msg("object", json!({ "w": 1 }))), + Kind::Work + )); + + st.stage.destinations[0].requires_stamp = false; + assert!(matches!( + classify(&st, &msg("linear", json!("shape is preserved"))), + Kind::Work + )); + } + #[test] fn the_stamp_never_replaces_a_payload_it_cannot_carry() { let st = runtime(vec![budget("b", 10)], 1.0); diff --git a/crates/server/src/store.rs b/crates/server/src/store.rs index 2ce7c9c..8f9ec2b 100644 --- a/crates/server/src/store.rs +++ b/crates/server/src/store.rs @@ -60,24 +60,112 @@ pub async fn save(queen: &Queen, doc: &GraphDoc) -> Result<()> { Ok(()) } +/// The declaration currently stored for one graph. +/// +/// A caller's declare needs this exact read even when the replica has not +/// reconciled yet. A prefix scan is the wrong primitive for that check: it can +/// be paged, and a graph past the first page would look new and escape the +/// version-bump rule. +pub async fn load_one(queen: &Queen, app: &str, name: &str) -> Result> { + let current = queen.kv().get(&ns(), &graph_key(app, name)).await?; + if current.found() { + let value = current.value.ok_or_else(|| { + queen_mq::Error::Decode(format!( + "stored graph `{app}/{name}` was found without a value" + )) + })?; + return decode_graph(value, app, name).map(Some); + } + + // A v1 standalone target may not have been restored by this replica yet. + // It is still the stored predecessor of the one-node graph the caller is + // about to replace, so it participates in the same version check. + let legacy = queen.kv().get(&ns(), &v1_target_key(app, name)).await?; + if !legacy.found() { + return Ok(None); + } + let value = legacy.value.ok_or_else(|| { + queen_mq::Error::Decode(format!( + "stored v1 target `{app}/{name}` was found without a value" + )) + })?; + let old: v1::TargetSpec = serde_json::from_value(value).map_err(|e| { + queen_mq::Error::Decode(format!( + "stored v1 target `{app}/{name}` is unreadable: {e}" + )) + })?; + gate_core::migrate::from_v1_target(&old) + .map(|m| Some(m.doc)) + .map_err(|e| queen_mq::Error::Decode(e.0)) +} + +fn decode_graph(value: serde_json::Value, app: &str, name: &str) -> Result { + match serde_json::from_value::(value.clone()) { + Ok(doc) => Ok(doc), + Err(v2_error) => { + let old: v1::GraphSpec = serde_json::from_value(value).map_err(|v1_error| { + queen_mq::Error::Decode(format!( + "stored graph `{app}/{name}` is neither a readable v2 graph ({v2_error}) nor \ + a readable v1 graph ({v1_error})" + )) + })?; + gate_core::migrate::from_v1_graph(&old) + .map(|m| m.doc) + .map_err(|e| queen_mq::Error::Decode(e.0)) + } + } +} + pub async fn forget(queen: &Queen, app: &str, name: &str) -> Result<()> { + // Remove legacy target rows first. A v1 graph-node target named + // `airbnb.ip` migrates to the one-node graph `ip`; deleting only + // `spec:{app}:ip` leaves `spec:{app}:airbnb.ip` behind and the next boot + // restores the graph that the caller just deleted. queen .kv() - .delete(&ns(), &graph_key(app, name)) + .delete(&ns(), &v1_target_key(app, name)) .send() .await?; - // A graph that came across from a v1 standalone target keeps its old row - // until it is deleted too, or the next boot restores it and the delete looks - // like it did not take. + forget_dotted_v1_targets(queen, app, name).await?; + queen .kv() - .delete(&ns(), &v1_target_key(app, name)) + .delete(&ns(), &graph_key(app, name)) .send() - .await - .ok(); + .await?; Ok(()) } +async fn forget_dotted_v1_targets(queen: &Queen, app: &str, graph: &str) -> Result<()> { + let prefix = format!("{V1_TARGET_PREFIX}{app}:"); + let found = queen + .kv() + .get_prefix(&ns(), &prefix) + .limit(1000) + .send() + .await?; + if found.truncated() { + return Err(queen_mq::Error::Invalid(format!( + "cannot safely delete `{app}/{graph}`: more than 1000 legacy target rows share this application" + ))); + } + + for row in found.rows.unwrap_or_default() { + let belongs = row + .value + .and_then(|value| serde_json::from_value::(value).ok()) + .is_some_and(|old| legacy_graph_name(&old.name) == graph); + if belongs { + queen.kv().delete(&ns(), &row.key).send().await?; + } + } + Ok(()) +} + +fn legacy_graph_name(target: &str) -> &str { + target.rsplit('.').next().unwrap_or(target) +} + /// What a prefix read found, and whether it found everything. /// /// The distinction is the difference between "nobody declared that" and "we did @@ -117,16 +205,9 @@ pub async fn try_load_all(queen: &Queen) -> Result { let mut items = Vec::new(); let mut migrated = Vec::new(); let mut unreadable = 0usize; - let mut truncated = false; - let res = queen - .kv() - .get_prefix(&ns(), GRAPH_PREFIX) - .limit(1000) - .send() - .await?; - truncated |= res.truncated(); - for row in res.rows.unwrap_or_default() { + let (rows, graphs_complete) = scan_prefix(queen, GRAPH_PREFIX).await?; + for row in rows { let Some(value) = row.value else { unreadable += 1; continue; @@ -161,14 +242,8 @@ pub async fn try_load_all(queen: &Queen) -> Result { } // v1 standalone targets. Each becomes a one-node graph named for itself. - let res = queen - .kv() - .get_prefix(&ns(), V1_TARGET_PREFIX) - .limit(1000) - .send() - .await?; - truncated |= res.truncated(); - for row in res.rows.unwrap_or_default() { + let (rows, targets_complete) = scan_prefix(queen, V1_TARGET_PREFIX).await?; + for row in rows { let Some(value) = row.value else { unreadable += 1; continue; @@ -202,7 +277,147 @@ pub async fn try_load_all(queen: &Queen) -> Result { } Ok(Stored { items, - complete: !truncated && unreadable == 0, + complete: graphs_complete && targets_complete && unreadable == 0, migrated, }) } + +/// Read every page under one store prefix. +/// +/// The broker caps a page by both rows and bytes. `truncated` therefore does +/// not mean merely "there may be more than 1,000 documents": one large value +/// can make even a short page incomplete. The exclusive `nextAfter` cursor is +/// the only correct way to resume it. +/// +/// A malformed truncated response is returned as incomplete rather than spun +/// on forever. Reconcile may still add/change the documents it did see, but its +/// `complete` guard will not interpret an unseen one as deleted. +async fn scan_prefix(queen: &Queen, prefix: &str) -> Result<(Vec, bool)> { + let namespace = ns(); + let mut rows = Vec::new(); + let mut after: Option = None; + + loop { + let mut query = queen.kv().get_prefix(&namespace, prefix).limit(1000); + if let Some(cursor) = &after { + query = query.after(cursor); + } + let page = query.send().await?; + let truncated = page.truncated(); + let next = page.next_after.clone(); + rows.extend(page.rows.unwrap_or_default()); + + if !truncated { + return Ok((rows, true)); + } + match next { + Some(cursor) if after.as_ref().is_none_or(|previous| cursor > *previous) => { + after = Some(cursor); + } + _ => return Ok((rows, false)), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use axum::extract::State; + use axum::routing::post; + use axum::{Json, Router}; + use parking_lot::Mutex; + use queen_mq::{Config, Queen}; + use serde_json::{json, Value}; + + use super::*; + + fn document(name: &str) -> Value { + json!({ + "application": "paging", + "graph": name, + "version": 1, + "nodes": { + "n": { + "budgets": [{ "id": "b", "count": 10, "timeMs": 1000 }], + "ingress": true, + "egress": "paging.out" + } + }, + "paths": [{ "name": "main", "nodes": ["n"] }] + }) + } + + async fn kv_page( + State(seen): State>>>, + Json(body): Json, + ) -> Json { + seen.lock().push(body.clone()); + let op = &body["operations"][0]; + let prefix = op["prefix"].as_str().unwrap_or_default(); + let after = op.get("after").and_then(Value::as_str); + let result = match (prefix, after) { + (GRAPH_PREFIX, None) => json!({ + "index": 0, + "op": "getPrefix", + "rows": [{ "key": "graph:paging:a", "value": document("a"), "version": 1 }], + "truncated": true, + "nextAfter": "graph:paging:a" + }), + (GRAPH_PREFIX, Some("graph:paging:a")) => json!({ + "index": 0, + "op": "getPrefix", + "rows": [{ "key": "graph:paging:b", "value": document("b"), "version": 1 }], + "truncated": false + }), + (V1_TARGET_PREFIX, None) => json!({ + "index": 0, + "op": "getPrefix", + "rows": [], + "truncated": false + }), + _ => panic!("unexpected prefix page: {op}"), + }; + Json(json!({ "results": [result] })) + } + + #[tokio::test] + async fn a_truncated_store_scan_resumes_from_the_brokers_cursor() { + let seen = Arc::new(Mutex::new(Vec::new())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fake broker"); + let url = format!("http://{}", listener.local_addr().expect("address")); + let router = Router::new() + .route("/api/v1/kv", post(kv_page)) + .with_state(seen.clone()); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("serve fake broker") + }); + let queen = Queen::connect(Config::new(url)).expect("client"); + + let stored = try_load_all(&queen).await.expect("scan store"); + server.abort(); + + assert!(stored.complete, "both prefixes reached their final page"); + assert_eq!( + stored.items.iter().map(GraphDoc::key).collect::>(), + ["paging/a", "paging/b"] + ); + let requests = seen.lock(); + assert_eq!(requests.len(), 3, "two graph pages and one target page"); + assert_eq!( + requests[1]["operations"][0]["after"], + json!("graph:paging:a"), + "the second page must use the broker's exclusive cursor" + ); + } + + #[test] + fn a_dotted_v1_target_maps_to_its_leaf_graph() { + assert_eq!(legacy_graph_name("airbnb.ip"), "ip"); + assert_eq!(legacy_graph_name("standalone"), "standalone"); + } +} diff --git a/crates/server/src/supervisor.rs b/crates/server/src/supervisor.rs index 4ad1afc..9d9ca29 100644 --- a/crates/server/src/supervisor.rs +++ b/crates/server/src/supervisor.rs @@ -59,13 +59,14 @@ pub async fn start( })); } + let stopped = Arc::new(AtomicBool::new(false)); let rt = Arc::new(GraphRuntime { doc, plan, stages, handles: parking_lot::RwLock::new(Vec::new()), persisted: AtomicBool::new(false), - stopped: AtomicBool::new(false), + stopped: stopped.clone(), cancel, }); @@ -80,6 +81,7 @@ pub async fn start( budgets.clone(), st.clone(), traces.clone(), + stopped.clone(), )); } *rt.handles.write() = handles; @@ -113,14 +115,13 @@ async fn provision(queen: &Queen, plan: &Plan) -> Result<()> { .configure(opts) .await?; } - // The application's queue. Created so its consumers can subscribe - // before Gate has pushed anything — a group that finds no queue is a - // 404 an application has to code around — and never configured: its - // retention, its lease and its partition count belong to whoever - // made it. - QueueKind::Egress => { - queen.queue(&q.name).create().await.ok(); - } + // Produced into, never configured. `QueueBuilder::create` is a + // `/configure` with an empty option bag, and that endpoint is a FULL + // replace: calling it here would silently reset an application's + // retention, lease, retry and dedup settings on every declare. Queen + // creates an absent queue atomically on the first push; applications + // that need to subscribe before then own its explicit provisioning. + QueueKind::Egress => {} // Consumed, never created. If it does not exist yet, Gate finds it // on its first message; declare-time validation says so as a // warning. diff --git a/crates/server/tests/live.rs b/crates/server/tests/live.rs index 776e494..f2e820d 100644 --- a/crates/server/tests/live.rs +++ b/crates/server/tests/live.rs @@ -72,7 +72,7 @@ use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use gate_server::api; -use queen_mq::{Config, Message, Queen, SubscriptionMode}; +use queen_mq::{Config, Expiry, Message, Queen, SubscriptionMode}; use serde_json::{json, Value}; fn queen_url() -> Option { @@ -193,25 +193,27 @@ async fn sweep(queen: &Queen) { if ONCE.set(()).is_err() { return; } - let Ok(res) = queen - .kv() - .get_prefix("gate", "graph:it") - .limit(1000) - .keys_only() - .send() - .await - else { - return; - }; - let rows = res.rows.unwrap_or_default(); - for row in &rows { - let _ = queen.kv().delete("gate", &row.key).send().await; - } - if !rows.is_empty() { - eprintln!( - "swept {} leftover graph document(s) from an earlier run", - rows.len() - ); + for prefix in ["graph:it", "spec:it"] { + let Ok(res) = queen + .kv() + .get_prefix("gate", prefix) + .limit(1000) + .keys_only() + .send() + .await + else { + return; + }; + let rows = res.rows.unwrap_or_default(); + for row in &rows { + let _ = queen.kv().delete("gate", &row.key).send().await; + } + if !rows.is_empty() { + eprintln!( + "swept {} leftover {prefix} document(s) from an earlier run", + rows.len() + ); + } } } @@ -410,16 +412,21 @@ impl Harness { /// v2 adds two failures it must cover that v1 had no equivalent of: a KV route /// that refuses (does the relay refund and release, or lose the batch?) and a /// transaction that fails AFTER a successful charge (does the refund fire?). +type RefusalRule = Arc, String)>>>; + struct FaultyBroker { url: String, - refuse: Arc>>, + refuse: RefusalRule, absent: Arc>>, seen: Arc>>, } impl FaultyBroker { fn refuse(&self, marker: &str) { - *self.refuse.write() = Some(marker.to_string()); + *self.refuse.write() = Some((None, marker.to_string())); + } + fn refuse_method(&self, method: axum::http::Method, marker: &str) { + *self.refuse.write() = Some((Some(method), marker.to_string())); } fn allow(&self) { *self.refuse.write() = None; @@ -446,7 +453,7 @@ impl FaultyBroker { struct ProxyState { real: String, http: reqwest::Client, - refuse: Arc>>, + refuse: RefusalRule, absent: Arc>>, seen: Arc>>, } @@ -518,9 +525,11 @@ async fn proxy( } } - if let Some(marker) = st.refuse.read().clone() { + if let Some((method, marker)) = st.refuse.read().clone() { let text = String::from_utf8_lossy(&bytes); - if marker.is_empty() || text.contains(&marker) || path.contains(&marker) { + let method_matches = method.as_ref().is_none_or(|m| m == parts.method); + if method_matches && (marker.is_empty() || text.contains(&marker) || path.contains(&marker)) + { return ( axum::http::StatusCode::INTERNAL_SERVER_ERROR, "refused by the test", @@ -590,6 +599,57 @@ fn egress_of(tag: &str, application: &str) -> String { format!("test.{tag}.{application}.out") } +/// Every flat route resolves a bare graph name across applications and refuses +/// to guess when more than one matches. DELETE used to be the exception: it +/// silently fell back to `default` and could remove that tenant's graph. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_flat_delete_refuses_an_ambiguous_graph_name() { + let Some(h) = harness("ambiguous-delete").await else { + return; + }; + let other = format!("{}-other", h.application); + let name = "same"; + + for (application, suffix) in [(&h.application, "one"), (&other, "two")] { + let doc = one_node( + &format!("test.ambiguous-delete.{application}.{suffix}.out"), + wide("b"), + ); + let (status, body) = h + .send( + reqwest::Method::PUT, + &format!("/v1/apps/{application}/graphs/{name}"), + Some(doc), + ) + .await; + assert_eq!(status, 200, "declare {application}: {body}"); + } + + let (status, body) = h + .send(reqwest::Method::DELETE, &format!("/v1/graphs/{name}"), None) + .await; + assert_eq!(status, 409, "an ambiguous delete must not choose: {body}"); + + for application in [&h.application, &other] { + let (status, body) = h + .send( + reqwest::Method::GET, + &format!("/v1/apps/{application}/graphs/{name}"), + None, + ) + .await; + assert_eq!(status, 200, "{application} was removed: {body}"); + let _ = h + .send( + reqwest::Method::DELETE, + &format!("/v1/apps/{application}/graphs/{name}"), + None, + ) + .await; + } +} + // ============================================================== the relay /// Exactly once, across a two-node graph. `got.len() == N` AND `distinct == N`, @@ -1045,7 +1105,7 @@ async fn a_failed_transaction_after_a_successful_charge_refunds() { } tokio::time::sleep(Duration::from_secs(6)).await; - let key = h.key("g", "n", "b"); + let key = gate_core::plan::shared_budget_key(&h.application, "vendor"); let spent = h.counter(&key).await; assert_eq!( spent, 0, @@ -1423,6 +1483,231 @@ async fn an_item_that_can_never_be_admitted_is_dead_lettered() { h.cleanup("g").await; } +/// A rule added after a document was stored must not take that document down. +/// +/// `restore` and `reconcile` declare through the same path a caller does, so a +/// refusal there does not keep anybody safe: it unregisters the graph, answers +/// 404 to its pushes, and leaves its ingress queue filling behind one WARN +/// line. The document was accepted by an older build and is serving traffic, so +/// it keeps running and the breach is logged. A CALLER's declare of the same +/// document is still refused, which is what the rule is for. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_stored_document_that_breaks_a_later_rule_keeps_running() { + let Some(h) = harness("grandfather").await else { + return; + }; + let out = egress_of("grandfather", &h.application); + // Only a per-key budget, which `node-unscoped-budget` refuses. Any rule that + // is not about naming or emptiness would do; this one has been enforced + // since v2 shipped, so the test needs no future rule to exist. + let doc = json!({ + "version": 1, + "nodes": { "n": { + "ingress": true, + "egress": out, + "budgets": [{ "id": "per-listing", "count": 100, "timeMs": 1000, + "scopeBy": "payload.listingId" }] + }}, + "paths": [{ "name": "main", "nodes": ["n"] }] + }); + + let (status, body) = h.put_graph("g", doc.clone()).await; + assert_eq!( + status, 422, + "a caller's declare must still be refused: {body}" + ); + assert!( + h.app.registry.get(&h.application, "g").is_none(), + "a refused declare must not register anything" + ); + + // The same document, already in the store — where an older build left it. + let mut stored: gate_core::GraphDoc = + serde_json::from_value(doc).expect("the document parses; only the rules refuse it"); + stored.application = h.application.clone(); + stored.graph = "g".into(); + gate_server::store::save(&h.queen, &stored) + .await + .expect("plant the stored document"); + + gate_server::reconcile(&h.app).await; + let rt = h.app.registry.get(&h.application, "g"); + assert!( + rt.is_some(), + "the stored graph was taken down by a rule added after it was written" + ); + assert!( + rt.expect("registered").is_running(), + "the stored graph was registered but never started" + ); + + h.cleanup("g").await; +} + +/// A non-object payload cannot carry `_gate.path`. Letting one enter an +/// interior queue read by several path groups makes the compiler's arbitrary +/// unstamped owner route and charge it as the wrong path. It must be +/// dead-lettered while its path is still unambiguous, without blocking valid +/// work behind it. A scalar remains legal on linear and terminal routes; this +/// test exercises only the shared destination that needs provenance. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn an_unstampable_item_never_enters_a_shared_interior_queue() { + let Some(h) = harness("unstampable").await else { + return; + }; + let out = egress_of("unstampable", &h.application); + let left = format!("app.unstampable.{}.left", h.application); + let right = format!("app.unstampable.{}.right", h.application); + h.queen.queue(&left).create().await.ok(); + h.queen.queue(&right).create().await.ok(); + + let doc = json!({ + "version": 1, + "nodes": { + "left": { + "ingress": { "queue": left, "http": false }, + "budgets": [wide("left")] + }, + "right": { + "ingress": { "queue": right, "http": false }, + "budgets": [wide("right")] + }, + "join": { "budgets": [wide("join")], "egress": out } + }, + "paths": [ + { "name": "left", "nodes": ["left", "join"] }, + { "name": "right", "nodes": ["right", "join"] } + ] + }); + let (status, body) = h.put_graph("g", doc).await; + assert_eq!(status, 200, "declare: {body}"); + + h.queen + .queue(&left) + .push_items(vec![ + queen_mq::PushItem { + queue: left.clone(), + partition: Some("p0".into()), + payload: json!("cannot carry a path stamp"), + transaction_id: None, + }, + queen_mq::PushItem { + queue: left.clone(), + partition: Some("p0".into()), + payload: json!({ "n": 1 }), + transaction_id: None, + }, + ]) + .await + .expect("push"); + + let got = h.drain(&out, 1, Duration::from_secs(40)).await; + assert_eq!(got.len(), 1, "valid work behind the poison never arrived"); + assert_eq!(got[0].data["n"], 1, "the unstampable item was forwarded"); + assert!( + h.drain_for(&out, Duration::from_secs(2)).await.is_empty(), + "the shared queue emitted another copy" + ); + + let (_, view) = h.get_graph("g").await; + let dead = view["stages"] + .as_array() + .and_then(|stages| { + stages + .iter() + .find(|stage| stage["path"] == "left" && stage["node"] == "left") + }) + .and_then(|stage| stage["counters"]["deadlettered"].as_u64()) + .unwrap_or(0); + assert!( + dead >= 1, + "the rejected item was not visible as dead-lettered: {view}" + ); + + h.cleanup("g").await; +} + +/// A sync of TARGETS does not delete a graph, and the runtime is what decides +/// that — not the store's copy of it. +/// +/// A redeclare registers before it saves. A graph that grew nodes and whose save +/// then failed is a one-node document in the store and a multi-node graph on +/// this replica, so an inventory built from the store alone offers it up for +/// reaping. Master never could: it iterated runtimes and skipped the wide ones. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_target_sync_never_reaps_a_graph_that_grew_nodes() { + let Some(h) = harness("wide-reap").await else { + return; + }; + let out = egress_of("wide-reap", &h.application); + + // What the store holds: the one-node shape, as it was first declared. + let (status, body) = h.put_graph("wide", one_node(&out, wide("b"))).await; + assert_eq!(status, 200, "declare: {body}"); + + // What this replica runs: two nodes, registered without the store agreeing. + let two = json!({ + "version": 2, + "nodes": { + "n": { "ingress": true, "budgets": [wide("b")] }, + "m": { "budgets": [wide("c")], "egress": out } + }, + "paths": [{ "name": "main", "nodes": ["n", "m"] }] + }); + let (status, body) = h.put_graph("wide", two).await; + assert_eq!(status, 200, "redeclare: {body}"); + gate_server::store::save( + &h.queen, + &serde_json::from_value({ + let mut d = one_node(&out, wide("b")); + d["application"] = json!(h.application); + d["graph"] = json!("wide"); + d + }) + .expect("document"), + ) + .await + .expect("put the store back to the one-node copy"); + + // A sync that names something else at all. + let (status, body) = h + .send( + reqwest::Method::PUT, + &format!("/v1/apps/{}/targets", h.application), + Some(json!([one_node(&out, wide("b")) + .as_object() + .map(|o| { + let mut o = o.clone(); + o.insert("graph".into(), json!("other")); + Value::Object(o) + }) + .expect("object")])), + ) + .await; + assert_eq!(status, 200, "sync: {body}"); + assert_eq!( + body["removed"], + json!([]), + "the graph is wider than a target and must not be reaped: {body}" + ); + + let rt = h.app.registry.get(&h.application, "wide"); + assert!( + rt.is_some(), + "the multi-node graph was stopped by a target sync" + ); + assert!( + rt.expect("registered").is_running(), + "it was left registered and stopped" + ); + + h.cleanup("wide").await; + h.cleanup("other").await; +} + /// A KV route that refuses is NOT a refusal. /// /// Reading a failed charge as a refusal would park the graph; reading it as an @@ -1979,6 +2264,58 @@ async fn a_path_added_to_a_running_graph_starts_at_the_tail() { // ============================================================== the breaker +/// A failed trip changes neither half of breaker state. +/// +/// The counter spend and the visible record used to be separate broker calls. +/// If the latter failed, the endpoint returned 502 while leaving a node held +/// until the TTL, with no breaker in the graph view to explain the outage. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_failed_breaker_trip_leaves_no_invisible_hold() { + let Some((h, faulty)) = faulty_harness("atomic-trip").await else { + assert!(std::env::var("GATE_TEST_REQUIRE_LIVE").is_err()); + return; + }; + let out = egress_of("atomic-trip", &h.application); + let (status, body) = h + .put_graph( + "g", + one_node(&out, json!({ "id": "b", "count": 1000, "timeMs": 1000 })), + ) + .await; + assert_eq!(status, 200, "declare: {body}"); + + let key = h.key("g", "n", "b"); + assert_eq!(h.counter(&key).await, 0, "the counter starts empty"); + + // `brk` occurs only in the breaker record key. Before the fix this lets the + // preceding counter-only call through and refuses the second, record-only + // call. With one batch it refuses the whole state transition. + faulty.refuse("brk"); + let (status, res) = h + .backoff("g", "n", json!({ "retryAfterSeconds": 30, "by": "test" })) + .await; + assert_eq!( + status, 502, + "the injected broker failure must surface: {res}" + ); + faulty.allow(); + + assert_eq!( + h.counter(&key).await, + 0, + "a failed trip must not leave the budget counter spent" + ); + let (status, view) = h.get_graph("g").await; + assert_eq!(status, 200, "{view}"); + assert!( + view["nodes"][0]["breaker"].is_null(), + "a failed trip must not publish a breaker either: {view}" + ); + + h.cleanup("g").await; +} + /// The breaker stops every path within one batch, and lifts on its own. /// /// A vendor's 429 becomes `POST .../backoff`, which SPENDS the node's window: the @@ -2144,64 +2481,331 @@ async fn one_owner_per_ingress_queue() { h.cleanup("second").await; } -/// The routes that are gone say where to go instead. +/// A sync that rejects one document is not a complete inventory and may not +/// delete an omitted target. /// -/// A 404 would read as "wrong URL" and send somebody hunting; a 410 with the -/// queue name and two lines of SDK is the difference between a migration and an -/// outage. +/// Returning `ok: false` after removing valid configuration is a destructive +/// partial success: a typo in one replacement document would turn into an +/// outage in an unrelated target before the caller could correct and retry it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] -async fn the_gone_routes_name_the_egress_queue() { - let Some(h) = harness("gone").await else { +async fn a_partially_refused_sync_reaps_nothing() { + let Some(h) = harness("sync-refusal").await else { return; }; - let out = egress_of("gone", &h.application); - let (status, body) = h.put_graph("g", one_node(&out, wide("b"))).await; - assert_eq!(status, 200, "declare: {body}"); + let out = egress_of("sync-refusal", &h.application); + // Two targets, so the list that follows is PARTIALLY valid and one target is + // omitted from it. A sync naming nothing valid would prove much less. + for name in ["keep", "drop"] { + let (status, body) = h + .put_graph(name, one_node(&format!("{out}.{name}"), wide("b"))) + .await; + assert_eq!(status, 200, "initial declare of {name}: {body}"); + } - let (status, res) = h + let mut valid = one_node(&format!("{out}.keep"), wide("b")); + valid["application"] = json!(h.application); + valid["graph"] = json!("keep"); + let (status, result) = h .send( - reqwest::Method::GET, - &format!("/v1/apps/{}/graphs/g/nodes/n/next?batch=10", h.application), - None, + reqwest::Method::PUT, + &format!("/v1/apps/{}/targets", h.application), + Some(json!([ + valid, + { + "application": h.application, + "graph": "broken", + "version": 1, + "nodes": {}, + "paths": [] + } + ])), ) .await; - assert_eq!(status, 410, "{res}"); + assert_eq!(status, 200, "sync response: {result}"); + assert_eq!(result["ok"], json!(false), "the invalid graph must fail"); + assert_eq!( + result["applied"], + json!(["keep"]), + "a valid document in the list still applies: {result}" + ); + assert_eq!( + result["removed"], + json!([]), + "a partial sync may reap nothing" + ); + + // `drop` is the one the caller left out. A refusal anywhere in the list is + // what makes the list unfit to authorise a deletion. + for name in ["keep", "drop"] { + let (status, view) = h.get_graph(name).await; + assert_eq!( + status, 200, + "`{name}` was deleted by a refused sync: {view}" + ); + assert!(view["running"].as_bool().unwrap_or(false), "{name}: {view}"); + } + + h.cleanup("keep").await; + h.cleanup("drop").await; +} + +/// A complete target inventory is authoritative even when it reaches a replica +/// that has not reconciled the stored targets into its local registry yet. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_fresh_replica_reaps_targets_from_the_stored_inventory() { + let Some(h) = harness("sync-store").await else { + return; + }; + let out = egress_of("sync-store", &h.application); + let (status, body) = h.put_graph("old", one_node(&out, wide("b"))).await; + assert_eq!(status, 200, "initial declare: {body}"); + + // The second replica deliberately knows no runtimes. An empty sync still + // means this application owns no standalone targets, not merely "remove + // whichever targets this process happens to have seen". + let second = serve(&h.app.queen_url).await; + assert!(second.registry.all().is_empty()); + let second_base = spawn_server(second).await; + let res = reqwest::Client::new() + .put(format!("{second_base}/v1/apps/{}/targets", h.application)) + .json(&json!([])) + .send() + .await + .expect("sync on fresh replica"); + let status = res.status().as_u16(); + let body: Value = res.json().await.unwrap_or(Value::Null); + assert_eq!(status, 200, "sync response: {body}"); + assert_eq!(body["ok"], json!(true), "sync response: {body}"); + assert_eq!(body["removed"], json!(["old"]), "sync response: {body}"); + + gate_server::reconcile(&h.app).await; assert!( - res["error"].as_str().unwrap_or_default().contains(&out), - "the headstone must name the queue: {res}" + h.app.registry.get(&h.application, "old").is_none(), + "the omitted target survived in the durable store" ); +} - let (status, res) = h - .send(reqwest::Method::POST, "/v1/leases/ack", Some(json!({}))) - .await; - assert_eq!(status, 410, "{res}"); +/// Gate-owned interior queues participate in the same fleet-wide ownership +/// rule as declared ingress queues. Calling one a "user ingress" in another +/// graph must not create a second consumer that forwards every internal frame. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn another_graph_cannot_claim_an_interior_queue_as_its_ingress() { + let Some(h) = harness("interior-owner").await else { + return; + }; + let mut owner = chain_doc(); + owner["nodes"]["ip"]["egress"] = json!(egress_of("interior-owner-a", &h.application)); + let (status, body) = h.put_graph("first", owner).await; + assert_eq!(status, 200, "declare owner: {body}"); + + let interior = gate_core::plan::interior_queue(&h.application, "first", "ip"); + let borrower = json!({ + "version": 1, + "nodes": { + "n": { + "ingress": { "queue": interior }, + "budgets": [wide("b")], + "egress": egress_of("interior-owner-b", &h.application) + } + }, + "paths": [{ "name": "main", "nodes": ["n"] }] + }); + let (status, refused) = h.put_graph("second", borrower).await; assert!( - res["error"] + status == 409 || status == 422, + "an interior queue must keep its one owner, got {status}: {refused}" + ); + assert!( + refused["error"] .as_str() .unwrap_or_default() - .contains("backoff"), - "and point at what replaced it: {res}" + .contains("already the source"), + "the refusal must identify the ownership collision: {refused}" ); - h.cleanup("g").await; + h.cleanup("first").await; + h.cleanup("second").await; } -/// A v1 document is accepted, mapped, and answered 200 with warnings naming -/// every field that was mapped or ignored — never a silent success, and never a -/// 422 for having been written last year. +/// Unknown remote ownership is not the same thing as an unowned queue. +/// +/// The local registry is deliberately empty: this is the replica on which the +/// old best-effort store scan silently turned both a transport error and an +/// incomplete inventory into permission to start a second consumer group. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] -async fn a_v1_target_is_accepted_and_mapped() { - let Some(h) = harness("v1").await else { return }; - - let (status, body) = h - .send( - reqwest::Method::PUT, - &format!("/v1/apps/{}/targets/legacy", h.application), - Some(json!({ - "version": 1, - "budgets": [{ "id": "api", "cap": 3000, "periodSeconds": 60, +async fn a_fresh_replica_fails_closed_when_source_ownership_cannot_be_verified() { + let Some((h, faulty)) = faulty_harness("owner-unknown").await else { + assert!(std::env::var("GATE_TEST_REQUIRE_LIVE").is_err()); + return; + }; + let shared_in = format!("test.owner-unknown.{}.in", h.application); + let doc = |egress: &str| { + json!({ + "version": 1, + "nodes": { + "n": { "ingress": { "queue": shared_in }, "budgets": [wide("b")], "egress": egress } + }, + "paths": [{ "name": "main", "nodes": ["n"] }] + }) + }; + + let (status, body) = h.put_graph("first", doc("test.owner-unknown.a")).await; + assert_eq!(status, 200, "declare: {body}"); + + let second = serve(&faulty.url).await; + assert!(second.registry.all().is_empty()); + let second_base = spawn_server(second.clone()).await; + let client = reqwest::Client::new(); + + // A transport failure used to be swallowed by `if let Ok(stored)`. + faulty.refuse("getPrefix"); + let res = client + .put(format!( + "{second_base}/v1/apps/{}/graphs/second", + h.application + )) + .json(&doc("test.owner-unknown.b")) + .send() + .await + .expect("declare against unreadable inventory"); + faulty.allow(); + let status = res.status().as_u16(); + let body: Value = res.json().await.unwrap_or(Value::Null); + assert_eq!(status, 502, "the failed ownership read was ignored: {body}"); + assert!( + body.to_string().contains("exclusive ownership"), + "the refusal should name the safety property: {body}" + ); + assert!(second.registry.all().is_empty()); + + // An unreadable document is an equally incomplete inventory. It may be a + // newer Gate document whose source this build cannot decode. + let namespace = gate_server::budget::namespace(); + let corrupt_key = format!("graph:{}:newer", h.application); + h.queen + .kv() + .put( + &namespace, + &corrupt_key, + json!({ "fromAFutureGate": true }), + queen_mq::Expiry::forever(), + ) + .send() + .await + .expect("plant unreadable document"); + let res = client + .put(format!( + "{second_base}/v1/apps/{}/graphs/second", + h.application + )) + .json(&doc("test.owner-unknown.b")) + .send() + .await + .expect("declare against incomplete inventory"); + let status = res.status().as_u16(); + let body: Value = res.json().await.unwrap_or(Value::Null); + assert_eq!(status, 502, "the incomplete inventory was ignored: {body}"); + assert!( + body.to_string().contains("inventory is incomplete"), + "{body}" + ); + assert!(second.registry.all().is_empty()); + + // But a graph whose sources Gate NAMES ITSELF is not blocked by the same + // unreadable row. `gate.{app}.{graph}.{node}.in` cannot be minted by another + // graph key, so there is no ownership question for the missing document to + // be hiding an answer to — and `complete` is a fact about the whole + // namespace, so refusing here would take every tenant's declares down for + // one document written by a newer build. + let res = client + .put(format!( + "{second_base}/v1/apps/{}/graphs/owned", + h.application + )) + .json(&one_node("test.owner-unknown.c", wide("b"))) + .send() + .await + .expect("declare a graph Gate names the source of"); + let status = res.status().as_u16(); + let body: Value = res.json().await.unwrap_or(Value::Null); + assert_eq!( + status, 200, + "an unreadable row elsewhere blocked a graph it cannot collide with: {body}" + ); + + h.queen + .kv() + .delete(&namespace, &corrupt_key) + .send() + .await + .expect("remove unreadable document"); + h.cleanup("first").await; + h.cleanup("second").await; + h.cleanup("owned").await; +} + +/// The routes that are gone say where to go instead. +/// +/// A 404 would read as "wrong URL" and send somebody hunting; a 410 with the +/// queue name and two lines of SDK is the difference between a migration and an +/// outage. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn the_gone_routes_name_the_egress_queue() { + let Some(h) = harness("gone").await else { + return; + }; + let out = egress_of("gone", &h.application); + let (status, body) = h.put_graph("g", one_node(&out, wide("b"))).await; + assert_eq!(status, 200, "declare: {body}"); + + let (status, res) = h + .send( + reqwest::Method::GET, + &format!("/v1/apps/{}/graphs/g/nodes/n/next?batch=10", h.application), + None, + ) + .await; + assert_eq!(status, 410, "{res}"); + assert!( + res["error"].as_str().unwrap_or_default().contains(&out), + "the headstone must name the queue: {res}" + ); + + let (status, res) = h + .send(reqwest::Method::POST, "/v1/leases/ack", Some(json!({}))) + .await; + assert_eq!(status, 410, "{res}"); + assert!( + res["error"] + .as_str() + .unwrap_or_default() + .contains("backoff"), + "and point at what replaced it: {res}" + ); + + h.cleanup("g").await; +} + +/// A v1 document is accepted, mapped, and answered 200 with warnings naming +/// every field that was mapped or ignored — never a silent success, and never a +/// 422 for having been written last year. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_v1_target_is_accepted_and_mapped() { + let Some(h) = harness("v1").await else { return }; + + let (status, body) = h + .send( + reqwest::Method::PUT, + &format!("/v1/apps/{}/targets/legacy", h.application), + Some(json!({ + "version": 1, + "budgets": [{ "id": "api", "cap": 3000, "periodSeconds": 60, "alignment": "calendar", "confidence": "inferred" }], "lanes": [{ "name": "default", "cap": "ceiling", "concurrency": 8, "default": true }], @@ -2249,6 +2853,8 @@ async fn the_console_can_draw_what_is_running() { let out = egress_of("console", &h.application); let mut doc = chain_doc(); doc["nodes"]["ip"]["egress"] = json!(out); + doc["nodes"]["ip"]["budgets"][0]["source"] = json!("vendor limits page"); + doc["nodes"]["ip"]["budgets"][0]["asOf"] = json!("2026-08-20"); let (status, body) = h.put_graph("g", doc).await; assert_eq!(status, 200, "declare: {body}"); @@ -2264,6 +2870,15 @@ async fn the_console_can_draw_what_is_running() { assert_eq!(topo["edges"][0]["to"], "ip"); assert_eq!(topo["paths"][0]["name"], "main"); + let (status, detail) = h.get_graph("g").await; + assert_eq!(status, 200, "{detail}"); + let budget = &detail["nodes"] + .as_array() + .and_then(|nodes| nodes.iter().find(|n| n["node"] == "ip")) + .expect("ip node missing")["budgets"][0]; + assert_eq!(budget["source"], "vendor limits page", "{detail}"); + assert_eq!(budget["asOf"], "2026-08-20", "{detail}"); + let (status, graphs) = h.send(reqwest::Method::GET, "/api/graphs", None).await; assert_eq!(status, 200, "{graphs}"); assert!(graphs.as_array().is_some_and(|a| !a.is_empty())); @@ -2276,6 +2891,11 @@ async fn the_console_can_draw_what_is_running() { json!(true), "the broker health must be probed: {overview}" ); + assert_eq!( + overview["history_error"], + Value::Null, + "a healthy history must not report one: {overview}" + ); assert!( overview["admitted_per_sec"].is_null(), "without the counters stream this must be null, not a lifetime average: {overview}" @@ -2340,6 +2960,57 @@ async fn the_console_can_draw_what_is_running() { // ============================================================== lifecycle +/// Declaring a graph must not reconfigure the application's egress queue. +/// +/// Queen's `create()` is implemented as `/configure` with an empty option bag, +/// and `/configure` is a full replace rather than a patch. Calling it merely to +/// ensure the queue exists resets every setting the application chose. An absent +/// egress needs no eager setup: Queen creates it atomically on Gate's first push. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn declaring_a_graph_preserves_its_egress_queue_configuration() { + let Some(h) = harness("egress-config").await else { + return; + }; + let out = egress_of("egress-config", &h.application); + h.queen + .queue(&out) + .configure(queen_mq::QueueOptions { + lease_time: Some(127), + retry_limit: Some(41), + ..Default::default() + }) + .await + .expect("configure application-owned egress"); + + let (status, body) = h.put_graph("g", one_node(&out, wide("b"))).await; + assert_eq!(status, 200, "declare: {body}"); + + let detail = h + .queen + .admin() + .queue_detail(&out, &[]) + .await + .expect("read egress configuration"); + assert_eq!( + detail["queue"]["config"]["leaseTime"], + json!(127), + "Gate replaced the application's lease setting: {detail}" + ); + assert_eq!( + detail["queue"]["config"]["retryLimit"], + json!(41), + "Gate replaced the application's retry setting: {detail}" + ); + + h.cleanup("g").await; + h.queen + .queue(&out) + .delete() + .await + .expect("delete application-owned egress"); +} + /// A failed provisioning leaves the old document serving. /// /// Without the restore the graph is left stopped and still registered: it accepts @@ -2421,7 +3092,7 @@ async fn a_declare_that_cannot_be_stored_is_not_acknowledged() { // Refuse only the store write. It is a path-route `PUT /api/v1/kv/{ns}/{key}`, // so the key reaches the proxy URL-ENCODED and the marker has to be spelt the // way the wire spells it — `graph:` matches nothing. - faulty.refuse("graph%3A"); + faulty.refuse_method(axum::http::Method::PUT, "graph%3A"); let (status, res) = h.put_graph("g", one_node(&out, wide("b"))).await; faulty.allow(); assert_eq!( @@ -2462,7 +3133,7 @@ async fn a_declare_that_cannot_be_stored_is_not_acknowledged() { let mut v2 = one_node(&out, wide("b")); v2["version"] = json!(2); v2["nodes"]["n"]["budgets"][0]["count"] = json!(7); - faulty.refuse("graph%3A"); + faulty.refuse_method(axum::http::Method::PUT, "graph%3A"); let (status, res) = h.put_graph("g", v2).await; faulty.allow(); assert_eq!(status, 502, "{res}"); @@ -2483,6 +3154,60 @@ async fn a_declare_that_cannot_be_stored_is_not_acknowledged() { h.cleanup("g").await; } +/// An ambiguous store write cannot make a later delete resurrect the graph. +/// +/// A broker can commit a PUT and lose only its response. The caller correctly +/// gets an error in that case, but the local runtime cannot know that the +/// document is already durable and keeps its `persisted` marker false. A +/// reconcile that observes the exact document is the missing proof: if it does +/// not repair the marker, a subsequent delete from another replica is read as +/// "my first save never landed" and the deleted graph is written back. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn reconcile_confirms_an_ambiguous_save_before_honouring_a_remote_delete() { + let Some(h) = harness("persisted-proof").await else { + return; + }; + let out = egress_of("persisted-proof", &h.application); + let (status, body) = h.put_graph("g", one_node(&out, wide("b"))).await; + assert_eq!(status, 200, "declare: {body}"); + + // Model the only unknowable part of a lost response: the store has the + // exact document, while this replica believes its PUT failed. + let rt = h.app.registry.get(&h.application, "g").expect("serving"); + rt.persisted + .store(false, std::sync::atomic::Ordering::Relaxed); + + gate_server::reconcile(&h.app).await; + let rt = h.app.registry.get(&h.application, "g").expect("serving"); + assert!( + rt.persisted.load(std::sync::atomic::Ordering::Relaxed), + "observing the exact stored document must confirm persistence" + ); + + // A different replica deletes the durable document. This replica must now + // honour the deletion; with the stale marker it would save the graph again. + gate_server::store::forget(&h.queen, &h.application, "g") + .await + .expect("remote delete"); + gate_server::reconcile(&h.app).await; + assert!( + h.app.registry.get(&h.application, "g").is_none(), + "the deleted graph was resurrected" + ); + + let stored = gate_server::store::try_load_all(&h.queen) + .await + .expect("read store"); + assert!( + stored + .items + .iter() + .all(|doc| { doc.application != h.application || doc.graph != "g" }), + "the reconcile pass wrote the remotely deleted document back" + ); +} + /// A declare that cannot be RESTORED leaves nothing registered. /// /// The other half of the provisioning contract, and the failure the whole @@ -2505,9 +3230,10 @@ async fn a_declare_that_cannot_be_restored_leaves_nothing_registered() { assert_eq!(status, 200, "declare: {body}"); assert!(h.app.registry.get(&h.application, "g").is_some()); - // Nothing gets through now, so neither the new plan nor the old one can be - // provisioned. - faulty.refuse(""); + // No queue can be configured now, so neither the new plan nor the old one + // can be provisioned. Reads remain available: this test is about a failed + // swap, not about the predecessor check failing closed. + faulty.refuse("configure"); let mut v2 = one_node(&out, wide("b")); v2["version"] = json!(2); v2["nodes"]["n"]["budgets"][0]["count"] = json!(7); @@ -2646,6 +3372,60 @@ async fn deleting_a_graph_that_was_never_declared_is_a_success() { assert_eq!(res["registered"], json!(false), "{res}"); } +/// A v1 target may be qualified as `graph.node`, while its v2 graph identity is +/// the final segment. Deleting that migrated graph must remove the qualified +/// source row too, or the next restore brings it back from the dead. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn deleting_a_migrated_dotted_target_removes_its_v1_source() { + let Some(h) = harness("delete-v1-dotted").await else { + return; + }; + let old_key = format!("spec:{}:legacy.ip", h.application); + let old = json!({ + "application": h.application, + "name": "legacy.ip", + "version": 1, + "budgets": [{ + "id": "api", "cap": 1000, "periodSeconds": 60, + "alignment": "rolling", "confidence": "inferred" + }], + "cost": { "field": "cost", "default": 1, "max": 1 } + }); + h.queen + .kv() + .put("gate", &old_key, old, queen_mq::Expiry::forever()) + .send() + .await + .expect("seed the v1 target row"); + + gate_server::restore(&h.app).await; + assert!( + h.app.registry.get(&h.application, "ip").is_some(), + "the qualified v1 target must migrate to its leaf graph" + ); + + let (status, body) = h + .send( + reqwest::Method::DELETE, + &format!("/v1/apps/{}/targets/ip", h.application), + None, + ) + .await; + assert_eq!(status, 200, "delete the migrated target: {body}"); + + let stored = gate_server::store::try_load_all(&h.queen) + .await + .expect("read the store after delete"); + assert!( + stored + .items + .iter() + .all(|doc| doc.key() != format!("{}/ip", h.application)), + "the legacy source survived and would restore the deleted graph" + ); +} + /// A second replica converges on the stored document. /// /// A declare lands on ONE replica. Without the store and the reconcile the fleet @@ -2697,6 +3477,42 @@ async fn a_second_replica_converges_on_the_stored_document() { ); } +/// A caller cannot evade the version-bump rule by reaching a replica before +/// that replica's reconcile loop has loaded the graph. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn a_fresh_replica_checks_the_stored_version_before_redeclaring() { + let Some(h) = harness("remote-version").await else { + return; + }; + let out = egress_of("remote-version", &h.application); + let (status, body) = h.put_graph("g", one_node(&out, wide("b"))).await; + assert_eq!(status, 200, "declare: {body}"); + + // This replica deliberately has no local runtime. Renaming the budget + // re-founds its counter and therefore needs a bump above the stored v1. + let second = serve(&h.app.queen_url).await; + assert!(second.registry.all().is_empty()); + let second_base = spawn_server(second).await; + let mut changed = one_node(&out, wide("renamed")); + changed["version"] = json!(1); + let res = reqwest::Client::new() + .put(format!("{second_base}/v1/apps/{}/graphs/g", h.application)) + .json(&changed) + .send() + .await + .expect("declare on the fresh replica"); + let status = res.status().as_u16(); + let body: Value = res.json().await.unwrap_or(Value::Null); + assert_eq!(status, 409, "the stored predecessor was ignored: {body}"); + assert!( + body.to_string().contains("bump version above 1"), + "the refusal should identify the required version: {body}" + ); + + h.cleanup("g").await; +} + /// A replica converges on a redeclared graph instead of wedging. /// /// The version-bump rule is enforced for a CALLER's declare only, never for one @@ -2778,14 +3594,14 @@ async fn the_reconcile_loop_converges_a_second_replica_on_its_own() { // ============================================================== depth and eta -/// A depth the broker will not report falls back to the last one it did. +/// A depth the broker will not report is unavailable, not the last value it did. /// -/// An outage costs one round trip per TTL instead of one per caller: a console -/// polling every few seconds across a dozen graphs would otherwise hammer an -/// admin API that is already unhappy. +/// The failure is cached for one TTL, so an honest outage still costs one Queen +/// request (the client retries that request three times) rather than one request +/// per caller. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] -async fn a_depth_the_broker_will_not_report_falls_back_to_the_last_one() { +async fn a_depth_the_broker_will_not_report_never_becomes_zero_or_stale() { let Some((h, faulty)) = faulty_harness("depth").await else { assert!(std::env::var("GATE_TEST_REQUIRE_LIVE").is_err()); return; @@ -2799,16 +3615,30 @@ async fn a_depth_the_broker_will_not_report_falls_back_to_the_last_one() { .await .expect("push"); - let first: u64 = h.app.depths.pending(&h.queen, &queue).await.values().sum(); + let first: u64 = h + .app + .depths + .pending(&h.queen, &queue) + .await + .expect("initial depth") + .values() + .sum(); assert_eq!(first, 1); // Wait past the cache TTL, then refuse. tokio::time::sleep(Duration::from_secs(3)).await; faulty.refuse("/depth"); - let stale: u64 = h.app.depths.pending(&h.queen, &queue).await.values().sum(); + faulty.forget(); + for _ in 0..5 { + assert!( + h.app.depths.pending(&h.queen, &queue).await.is_err(), + "neither zero nor a stale depth is a live answer" + ); + } assert_eq!( - stale, 1, - "the last answer is served rather than a zero, which would read as an empty queue" + faulty.hits("/depth"), + 3, + "the failure must be cached after one client request and its retries" ); faulty.allow(); } @@ -2841,6 +3671,258 @@ async fn an_eta_against_an_older_broker_still_costs_one_probe_per_ttl() { faulty.allow(); } +/// A broker depth outage is not an empty graph or an immediate ETA. +/// +/// Every endpoint below used to consume the cache's default or stale map as if +/// it were a live answer. They now fail together, while the cached failure +/// keeps several page requests from repeating the same broken admin call. The +/// default Queen client makes three HTTP attempts for that one call. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn live_state_endpoints_do_not_invent_zero_during_a_depth_outage() { + let Some((h, faulty)) = faulty_harness("depthstate").await else { + assert!(std::env::var("GATE_TEST_REQUIRE_LIVE").is_err()); + return; + }; + let out = egress_of("depthstate", &h.application); + let (status, body) = h + .put_graph( + "g", + one_node(&out, json!({ "id": "b", "count": 1000, "timeMs": 1000 })), + ) + .await; + assert_eq!(status, 200, "declare: {body}"); + + // The declaration response populated the depth cache. Once it expires, a + // failed refresh must not resurrect that old value as if it were current. + tokio::time::sleep(Duration::from_secs(3)).await; + faulty.refuse("/depth"); + faulty.forget(); + let paths = [ + format!("/v1/apps/{}/graphs/g", h.application), + format!("/v1/apps/{}/graphs/g/nodes/n/eta", h.application), + "/api/targets".into(), + "/api/graphs".into(), + format!("/v1/apps/{}/metrics", h.application), + ]; + for path in paths { + let (status, body) = h.send(reqwest::Method::GET, &path, None).await; + assert_eq!( + status, 502, + "{path} invented a backlog while depth was unavailable: {body}" + ); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("live broker state"), + "{path}: {body}" + ); + } + assert_eq!( + faulty.hits("/depth"), + 3, + "the failed read must be cached across page endpoints after one client request" + ); + + faulty.allow(); + h.cleanup("g").await; +} + +/// Live state is either read from the broker or reported as unavailable. A KV +/// outage used to become an empty vector at every call site, which made the +/// same graph appear to have zero usage, no active breaker and an immediate ETA +/// while the source of truth was unreachable. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn live_state_endpoints_do_not_invent_zero_during_a_kv_outage() { + let Some((h, faulty)) = faulty_harness("stateread").await else { + assert!(std::env::var("GATE_TEST_REQUIRE_LIVE").is_err()); + return; + }; + let out = egress_of("stateread", &h.application); + let budget = json!({ + "id": "b", + "sharedKey": "vendor", + "count": 1000, + "timeMs": 1000 + }); + let (status, body) = h.put_graph("g", one_node(&out, budget)).await; + assert_eq!(status, 200, "declare: {body}"); + + faulty.refuse("/api/v1/kv"); + let paths = [ + format!("/v1/apps/{}/graphs/g", h.application), + format!("/v1/apps/{}/graphs/g/nodes/n/eta", h.application), + "/api/targets".into(), + "/api/budgets".into(), + "/api/breaches/recent".into(), + format!("/v1/apps/{}/metrics", h.application), + ]; + for path in paths { + let (status, body) = h.send(reqwest::Method::GET, &path, None).await; + assert_eq!( + status, 502, + "{path} invented live state while KV was unavailable: {body}" + ); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("live broker state"), + "{path}: {body}" + ); + } + + faulty.allow(); + h.cleanup("g").await; +} + +/// A present counter whose VALUE is not a number is corrupt state, not zero. +/// +/// The two halves of a counter row fail differently, on purpose. A value that +/// is not an integer cannot be reasoned about at all, so it is an error and the +/// charge gives back anything it can prove it wrote first. A missing or +/// unreadable EXPIRY costs only the park deadline, which degrades to "retry +/// now" — failing the charge over it would stop every path on that counter for +/// ever, because a failed charge is redelivered and fails again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs a broker: set GATE_TEST_QUEEN_URL and run with --include-ignored"] +async fn malformed_budget_state_is_reported_and_a_failed_decode_is_refunded() { + let Some(h) = harness("badbudgetstate").await else { + return; + }; + let out = egress_of("badbudgetstate", &h.application); + let budget = json!({ + "id": "b", + "sharedKey": "vendor", + "count": 100, + "timeMs": 1000 + }); + let (status, body) = h.put_graph("g", one_node(&out, budget)).await; + assert_eq!(status, 200, "declare: {body}"); + + let key = gate_core::plan::shared_budget_key(&h.application, "vendor"); + h.queen + .kv() + .put( + h.app.budgets.ns(), + &key, + json!("three"), + Expiry::seconds(60), + ) + .send() + .await + .expect("plant a counter whose value is not a number"); + + let paths = [ + format!("/v1/apps/{}/graphs/g", h.application), + format!("/v1/apps/{}/graphs/g/nodes/n/eta", h.application), + "/api/targets".into(), + "/api/budgets".into(), + format!("/v1/apps/{}/metrics", h.application), + ]; + for path in paths { + let (status, body) = h.send(reqwest::Method::GET, &path, None).await; + assert_eq!( + status, 502, + "{path} reported the corrupt counter as zero: {body}" + ); + assert!( + body["error"] + .as_str() + .unwrap_or_default() + .contains("live broker state"), + "{path}: {body}" + ); + } + + let error = h + .app + .budgets + .charge(&[gate_server::budget::Charge { + key: key.clone(), + max: 100, + ttl: 1, + delta: 2, + budget_id: "b".into(), + }]) + .await + .expect_err("a returned counter whose value is not an integer must fail decoding"); + assert!(error.to_string().contains("not an integer"), "{error}"); + let raw = h + .app + .budgets + .get_raw(std::slice::from_ref(&key)) + .await + .expect("read counter after refund"); + assert_eq!( + raw.first().and_then(|row| row.value.clone()), + Some(json!("three")), + "the applied increment must be refunded before the decode error escapes" + ); + + // The other half: a counter with a real value and no expiry is READABLE. + // It loses its deadline and nothing else, so the graph keeps admitting. + h.app + .budgets + .clear(std::slice::from_ref(&key)) + .await + .expect("remove corrupt counter"); + h.queen + .kv() + .put(h.app.budgets.ns(), &key, json!(3), Expiry::forever()) + .send() + .await + .expect("plant a counter with no window expiry"); + + let states = h + .app + .budgets + .read(std::slice::from_ref(&key)) + .await + .expect("a counter without an expiry is readable, not an error"); + assert_eq!(states.first().map(|s| s.value), Some(3), "{states:?}"); + assert_eq!( + states.first().and_then(|s| s.expires_at_ms), + None, + "an unreadable expiry degrades to `retry now`" + ); + + let attempt = h + .app + .budgets + .charge(&[gate_server::budget::Charge { + key: key.clone(), + max: 100, + ttl: 1, + delta: 2, + budget_id: "b".into(), + }]) + .await + .expect("a missing expiry must not fail the charge and stall the node"); + assert!( + attempt.all_applied(), + "the charge is decided by the counter, not by its deadline: {attempt:?}" + ); + + let (status, body) = h + .send( + reqwest::Method::GET, + &format!("/v1/apps/{}/graphs/g", h.application), + None, + ) + .await; + assert_eq!(status, 200, "a readable counter must not 502: {body}"); + + h.app + .budgets + .clear(std::slice::from_ref(&key)) + .await + .expect("remove the expiryless counter"); + h.cleanup("g").await; +} + /// An ETA answers from the DECLARED schedule when the window is spent. /// /// A window with nothing left in it measures zero admissions per second, and zero @@ -2895,6 +3977,20 @@ async fn an_eta_answers_from_the_declared_schedule_when_the_window_is_spent() { "the answer is a bound and must read as one: {eta}" ); + let (_, targets) = h.send(reqwest::Method::GET, "/api/targets", None).await; + let mine = targets + .as_array() + .and_then(|rows| { + rows.iter() + .find(|t| t["application"] == h.application && t["name"] == "g") + }) + .expect("graph missing from target index"); + assert_eq!( + mine["state"], "pacing", + "current backlog must drive state: {mine}" + ); + assert!(mine["backlog"].as_u64().unwrap_or(0) > 0, "{mine}"); + h.cleanup("g").await; } @@ -2945,6 +4041,30 @@ async fn an_eta_tells_a_budget_backlog_from_a_worker_one() { assert_eq!(eta["waitingForBudget"], json!(0), "{eta}"); assert_eq!(eta["state"], "waiting-workers", "{eta}"); + // The graph detail is what the topology diagram reads. These fields used + // to be absent, which the Vue component silently rendered as two zeroes. + let (status, view) = h.get_graph("g").await; + assert_eq!(status, 200, "{view}"); + let node = &view["nodes"][0]; + assert_eq!(node["waiting_for_budget"], json!(0), "{view}"); + assert!( + node["waiting_for_workers"].as_u64().unwrap_or(0) as usize >= N, + "the graph must show the worker backlog instead of a fallback zero: {view}" + ); + + let (_, targets) = h.send(reqwest::Method::GET, "/api/targets", None).await; + let mine = targets + .as_array() + .and_then(|rows| { + rows.iter() + .find(|t| t["application"] == h.application && t["name"] == "g") + }) + .expect("graph missing from target index"); + assert_eq!( + mine["state"], "flowing", + "worker backlog is not budget pacing: {mine}" + ); + h.cleanup("g").await; } @@ -3253,6 +4373,7 @@ async fn an_ack_settles_the_whole_claim_or_pays_a_lease() { .depths .pending_of_group(&h.queen, &q, "g") .await + .expect("group depth") .values() .sum(); assert_eq!(owed, 3, "nothing is lost: the group still owes the tail"); diff --git a/crates/server/tests/units.rs b/crates/server/tests/units.rs index f4cfab6..9939fa9 100644 --- a/crates/server/tests/units.rs +++ b/crates/server/tests/units.rs @@ -145,6 +145,24 @@ fn the_worker_count_comes_from_the_budget_and_not_from_the_partitions() { 16, "the per-key budget is not a rate" ); + + // Nor is a selector a node-wide rate. A rare operation capped at one per + // hour must not reduce unrelated traffic from four lanes to one. + let conditional = doc(json!({ + "application": "a", "graph": "g", "version": 1, + "nodes": { "n": { "ingress": { "queue": "theirs.in" }, "egress": "theirs.out", + "budgets": [ + { "id": "node", "count": 4000, "timeMs": 1000 }, + { "id": "rare", "count": 1, "timeMs": 3600000, + "subWindows": 1, "whenOp": ["photo.delete"] } + ] } }, + "paths": [{ "name": "main", "nodes": ["n"] }] + })); + assert_eq!( + with_partitions(&conditional, 16), + 4, + "a conditional budget is not the node's rate" + ); } /// A path's SHARE is part of its ceiling, so it is part of its worker count: a @@ -293,6 +311,23 @@ fn a_claim_is_sized_by_the_nodes_own_rate_and_never_by_a_per_key_one() { "a batch of two hundred messages across two hundred different keys spends one unit of \ each: sizing on a per-key allowance is how a node stops draining" ); + + let conditional = doc(json!({ + "application": "a", "graph": "g", "version": 1, + "nodes": { "n": { "ingress": true, + "budgets": [ + { "id": "node", "count": 500, "timeMs": 1000 }, + { "id": "rare", "count": 1, "timeMs": 3600000, + "subWindows": 1, "whenOp": ["photo.delete"] } + ], + "egress": "out" } }, + "paths": [{ "name": "main", "nodes": ["n"] }] + })); + assert_eq!( + gate_core::compile(&conditional).stages[0].batch, + gate_core::DEFAULT_BATCH, + "a selector for one operation must not shrink every claim" + ); } /// `null` rather than infinity when nothing is moving: "we cannot say" is an @@ -334,6 +369,17 @@ fn the_knobs_default_to_what_the_design_says() { k.retry_limit, 3, "the DLQ is back: v1 had to disarm it because it paced by nacking" ); + assert!( + k.max_push_body <= gate_server::knobs::MAX_PUSH_BODY_CEILING, + "the knob is floored at axum's default and capped: a per-request buffer is a memory \ + reservation, and nothing limits how many requests hold one at once" + ); + assert_eq!( + k.max_push_body, + 8 * 1024 * 1024, + "four times axum's 2 MiB default, which was the real ceiling on every push \ + until 2026-09-04 because nothing here ever set one" + ); } /// `forwarded / commits` is THE number that explains a stage's throughput: the @@ -376,5 +422,105 @@ fn the_trace_ring_drops_the_oldest_and_never_grows() { (gate_server::obs::TRACE_RING + 49) as i64, "newest first" ); + assert_eq!(recent[0].view()["budget_id"], "b"); + assert_eq!(recent[0].view()["budgetId"], "b", "compatibility alias"); assert!(t.recent(Some("admitted"), 10).is_empty(), "denials only"); } + +// ----------------------------------------------------------- the body limit + +/// A push carries a BATCH and everything else carries a document, so only the +/// push routes get the raised body limit. +/// +/// The limit is asserted through the ROUTER rather than by reading the knob back, +/// because the knob was never the thing that was wrong: axum applies a 2 MiB +/// default to every route unless a layer says otherwise, and for the life of this +/// service nothing did. A test that reads `knobs().max_push_body` would have +/// passed just as happily on the day a caller was being refused. +/// +/// No broker is needed and none is reached: a body over the limit is rejected by +/// the extractor, so the handler never runs. +mod body_limit { + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use std::sync::Arc; + use tower::ServiceExt; + + fn app() -> gate_server::api::Shared { + // Deliberately unreachable: nothing in these cases gets far enough to + // speak to it, and a test that needed a broker would belong in live.rs. + let queen = queen_mq::Queen::connect(queen_mq::Config::new("http://127.0.0.1:1")) + .expect("the client is constructed, not connected"); + Arc::new(gate_server::api::App::new( + queen, + "http://127.0.0.1:1".into(), + )) + } + + fn body_of(bytes: usize) -> Body { + Body::from(vec![b'x'; bytes]) + } + + const MIB: usize = 1024 * 1024; + + #[tokio::test] + async fn a_push_accepts_a_body_axums_default_would_have_refused() { + let res = gate_server::api::router(app()) + .oneshot( + Request::post("/v1/apps/channel-go/graphs/google/nodes/hotel/push") + .header("content-type", "application/json") + .body(body_of(3 * MIB)) + .unwrap(), + ) + .await + .unwrap(); + assert_ne!( + res.status(), + StatusCode::PAYLOAD_TOO_LARGE, + "a 3 MiB push was refused for its size: this is the 2 MiB default that \ + cost a caller a week of undelivered pushes, and raising it is the point" + ); + } + + #[tokio::test] + async fn a_push_over_the_ceiling_is_still_refused() { + let over = gate_server::knobs::knobs().max_push_body + MIB; + let res = gate_server::api::router(app()) + .oneshot( + Request::post("/v1/apps/channel-go/graphs/google/nodes/hotel/push") + .header("content-type", "application/json") + .body(body_of(over)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + res.status(), + StatusCode::PAYLOAD_TOO_LARGE, + "the ceiling is a ceiling: a body past it must be refused, or the limit \ + is memory nobody is bounding" + ); + } + + #[tokio::test] + async fn a_document_route_keeps_the_default() { + // Declaring a graph is a document, not a batch. Raising its ceiling would + // buy nothing and would let a caller hand a 512 MiB pod a body per + // request that no declaration has ever needed. + let res = gate_server::api::router(app()) + .oneshot( + Request::put("/v1/apps/channel-go/graphs/google") + .header("content-type", "application/json") + .body(body_of(3 * MIB)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + res.status(), + StatusCode::PAYLOAD_TOO_LARGE, + "a document route accepted 3 MiB: the raise must be scoped to the push \ + routes, not applied to the whole surface" + ); + } +} diff --git a/ui/package-lock.json b/ui/package-lock.json index 9a9b3cd..69748dc 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -18,6 +18,9 @@ "@vitejs/plugin-vue": "^5.2.1", "tailwindcss": "^4.3.3", "vite": "^6.0.7" + }, + "engines": { + "node": ">=24.0.0" } }, "node_modules/@babel/helper-string-parser": { @@ -1165,6 +1168,72 @@ "node": ">=14.0.0" } }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.1", + "dev": true, + "inBundle": true, + "license": "0BSD", + "optional": true + }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", @@ -1457,7 +1526,6 @@ "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -1468,7 +1536,6 @@ "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", "dev": true, "license": "MPL-2.0", - "peer": true, "dependencies": { "detect-libc": "^2.0.3" }, @@ -1902,7 +1969,6 @@ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -1916,7 +1982,6 @@ "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", @@ -2010,7 +2075,6 @@ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -2023,7 +2087,6 @@ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.41.tgz", "integrity": "sha512-2laE0p+aK+/AOPG/XL/WepOs/GlK755LJ1XECi9kDUrz1FKNw8rb2Xzlw9JS1rqEV55nb0ttsKxVlTCcd+R5cg==", "license": "MIT", - "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.41", "@vue/compiler-sfc": "3.5.41", diff --git a/ui/package.json b/ui/package.json index 1c84933..eda292d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -3,6 +3,9 @@ "version": "0.1.0", "private": true, "type": "module", + "engines": { + "node": ">=24.0.0" + }, "scripts": { "dev": "vite", "build": "vite build", diff --git a/ui/src/App.vue b/ui/src/App.vue index 403d31a..c2ba978 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -3,12 +3,15 @@ import { ref, onMounted, computed } from 'vue' import { useRoute } from 'vue-router' import Icon from './components/Icon.vue' import SignIn from './views/SignIn.vue' -import { api, authState, me, fetchMe, isAdmin, LOGOUT_URL, READ_ONLY_NOTE } from './lib/api.js' +import { api, authState, authError, me, fetchMe, isAdmin, READ_ONLY_NOTE } from './lib/api.js' +import { usePoll } from './lib/poll.js' const route = useRoute() const overview = ref(null) const dark = ref(document.documentElement.classList.contains('dark')) const mobileNav = ref(false) +const signingOut = ref(false) +const signOutError = ref('') /* Navigation grouped by intent: "Monitor" is what an operator opens when a @@ -37,6 +40,23 @@ function toggleTheme() { localStorage.setItem('gate-theme', dark.value ? 'dark' : 'light') } +async function signOut() { + if (signingOut.value) return + signingOut.value = true + signOutError.value = '' + try { + await api.post('/api/auth/logout', null) + window.location.assign('/') + } catch (e) { + // A 401 already switches the shell to SignIn. Other failures leave the + // current session intact and should be visible rather than becoming an + // unhandled event promise in the console. + if (authState.value !== 'login') signOutError.value = e.message + } finally { + signingOut.value = false + } +} + async function load() { if (authState.value !== 'ready') return try { @@ -45,10 +65,17 @@ async function load() { overview.value = null } } -onMounted(async () => { + +async function retryAuth() { + authState.value = 'unknown' await fetchMe() load() - setInterval(() => !document.hidden && load(), 15000) +} + +const refresh = usePoll(load, 15000) +onMounted(async () => { + await fetchMe() + refresh() }) const brokerOk = computed(() => overview.value?.queen?.reachable === true) @@ -81,6 +108,24 @@ const warnings = computed(() => { looks broken rather than closed. --> +
+
+ + + +

Console unavailable

+

+ Gate could not establish whether this session is signed in. +

+

+ {{ authError }} +

+ +
+
+
+

+ Could not sign out: {{ signOutError }} +

{{ READ_ONLY_NOTE }}

diff --git a/ui/src/components/BudgetBar.vue b/ui/src/components/BudgetBar.vue index d973d25..d5a0c63 100644 --- a/ui/src/components/BudgetBar.vue +++ b/ui/src/components/BudgetBar.vue @@ -22,13 +22,14 @@ import { computed } from 'vue' on-screen evidence that the model and the vendor disagree. */ const props = defineProps({ - used: { type: Number, required: true }, + used: { type: Number, default: null }, cap: { type: Number, required: true }, assumed: { type: Boolean, default: false }, height: { type: Number, default: 6 }, }) -const ratio = computed(() => (props.cap > 0 ? props.used / props.cap : 0)) +const known = computed(() => props.used !== null && props.used !== undefined && props.cap > 0) +const ratio = computed(() => (known.value ? props.used / props.cap : 0)) const width = computed(() => `${Math.min(100, Math.max(0, ratio.value * 100))}%`) const over = computed(() => ratio.value > 1) @@ -43,7 +44,7 @@ const fill = computed(() => {
-
+
+ no single live value is available for this budget +
over cap by {{ Math.round((ratio - 1) * 100) }}%
diff --git a/ui/src/components/FlowChart.vue b/ui/src/components/FlowChart.vue index 280f3fc..3067a29 100644 --- a/ui/src/components/FlowChart.vue +++ b/ui/src/components/FlowChart.vue @@ -34,19 +34,25 @@ const data = ref(undefined) const error = ref('') async function load() { + const minutes = range.value.minutes try { - const r = await api.get(`/api/flow?minutes=${range.value.minutes}`) + const r = await api.get(`/api/flow?minutes=${minutes}`) + if (minutes !== range.value.minutes) return data.value = r?.minutes?.length ? r : null error.value = '' } catch (e) { + if (minutes !== range.value.minutes) return error.value = e.message } } /* Slower than the live gauges on this page: the series is one point per minute, so refreshing it every four seconds would redraw the same picture fifteen times to add nothing. */ -usePoll(load, 15000) -watch(range, load) +const refresh = usePoll(load, 15000) +watch(range, () => { + data.value = undefined + refresh() +}) const series = computed(() => data.value?.applications ?? []) const minutes = computed(() => data.value?.minutes ?? []) @@ -221,7 +227,7 @@ function toneOf(u) { would read as a limit of zero, which is the opposite of idle. --> diff --git a/ui/src/components/TraceList.vue b/ui/src/components/TraceList.vue index 41b6a13..700ff45 100644 --- a/ui/src/components/TraceList.vue +++ b/ui/src/components/TraceList.vue @@ -25,6 +25,8 @@ const WORD = { function tone(t) { return t.outcome === 'throttled' ? 'text-bad' : t.outcome === 'ok' ? 'text-fg-3' : 'text-fg-2' } + +const traceBudget = (t) => t.budget_id ?? t.budgetId